diff --git a/CHANGELOG.md b/CHANGELOG.md index 615b9a7..037580b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,11 +5,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] - -### Added -- Professional repository baseline: - - CI workflow with lint, type checks, tests, and secret scanning gates. - - `pre-commit` with `ruff`, `mypy`, and `gitleaks`. - - `Makefile`, `CODEOWNERS`, and PR template. - - Initial `SECURITY.md` and packaging metadata improvements. +## [0.1.11] - 2026-06-15 + +### Changed + +- Relaxed bundled `tinybird` CLI dependency to `>=4.6.0,<4.7.0` to avoid resolution failures while keeping the SDK on the `4.6.x` line. +- Updated branch data config handling to use `branch_data_mode`; legacy `branch_data_on_create` now triggers an explicit migration error. +- `branch_data_mode` now only accepts `last_partition` as a user-facing value. +- In `dev_mode=local`, branch data mode warnings are now shown only when `branch_data_mode` is explicitly set in `tinybird.config.json`. +- `tinybird branch create` and `tinybird branch clear` now show a deprecation warning (instead of failing) when `--ignore-datasource` is passed, then continue by ignoring that flag. diff --git a/pyproject.toml b/pyproject.toml index 39bc80f..660ca16 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "tinybird-sdk" -version = "0.2.0" +version = "0.3.0" description = "Python SDK for Tinybird Forward" readme = "README.md" license = "MIT" @@ -9,7 +9,7 @@ authors = [ ] requires-python = ">=3.11" dependencies = [ - "tinybird==4.6.0", + "tinybird>=4.6.0,<4.7.0", ] classifiers = [ "Development Status :: 4 - Beta", diff --git a/src/tinybird_sdk/api/__init__.py b/src/tinybird_sdk/api/__init__.py index 799ffbe..96832c5 100644 --- a/src/tinybird_sdk/api/__init__.py +++ b/src/tinybird_sdk/api/__init__.py @@ -13,6 +13,7 @@ ) from .branches import ( TinybirdBranch, + CreateBranchOptions, BranchApiConfig, BranchApiError, create_branch, diff --git a/src/tinybird_sdk/api/branches.py b/src/tinybird_sdk/api/branches.py index 64f19e8..4facaab 100644 --- a/src/tinybird_sdk/api/branches.py +++ b/src/tinybird_sdk/api/branches.py @@ -1,15 +1,13 @@ from __future__ import annotations import time -from dataclasses import asdict -from dataclasses import dataclass +from dataclasses import asdict, dataclass from typing import Any from urllib.parse import urlencode +from ..cli.config_types import BranchDataMode from .fetcher import tinybird_fetch -LAST_PARTITION = "last_partition" - @dataclass(frozen=True, slots=True) class BranchApiConfig: @@ -25,6 +23,11 @@ class TinybirdBranch: token: str | None = None +@dataclass(frozen=True, slots=True) +class CreateBranchOptions: + branch_data_mode: BranchDataMode | None = None + + class BranchApiError(Exception): def __init__(self, message: str, status: int, body: Any = None): super().__init__(message) @@ -68,9 +71,14 @@ def _poll_job( raise BranchApiError(f"Job '{job_id}' timed out after {max_attempts} attempts", 408) -def create_branch(config: BranchApiConfig | dict[str, Any], name: str) -> TinybirdBranch: +def create_branch( + config: BranchApiConfig | dict[str, Any], name: str, options: CreateBranchOptions | None = None +) -> TinybirdBranch: normalized = config if isinstance(config, BranchApiConfig) else BranchApiConfig(**config) - url = f"{normalized.base_url.rstrip('/')}/v1/environments?{urlencode({'name': name})}" + params = {"name": name} + if options and options.branch_data_mode: + params["data"] = options.branch_data_mode + url = f"{normalized.base_url.rstrip('/')}/v1/environments?{urlencode(params)}" response = tinybird_fetch(url, method="POST", headers=_headers(normalized.token)) if not response.ok: @@ -152,19 +160,23 @@ def branch_exists(config: BranchApiConfig | dict[str, Any], name: str) -> bool: return any(branch.name == name for branch in branches) -def get_or_create_branch(config: BranchApiConfig | dict[str, Any], name: str) -> dict[str, Any]: +def get_or_create_branch( + config: BranchApiConfig | dict[str, Any], name: str, options: CreateBranchOptions | None = None +) -> dict[str, Any]: normalized = config if isinstance(config, BranchApiConfig) else BranchApiConfig(**config) try: branch = get_branch(normalized, name) return {**asdict(branch), "was_created": False} except BranchApiError as error: if error.status == 404: - branch = create_branch(normalized, name) + branch = create_branch(normalized, name, options=options) return {**asdict(branch), "was_created": True} raise -def clear_branch(config: BranchApiConfig | dict[str, Any], name: str) -> TinybirdBranch: +def clear_branch( + config: BranchApiConfig | dict[str, Any], name: str, options: CreateBranchOptions | None = None +) -> TinybirdBranch: normalized = config if isinstance(config, BranchApiConfig) else BranchApiConfig(**config) delete_branch(normalized, name) - return create_branch(normalized, name) + return create_branch(normalized, name, options=options) diff --git a/src/tinybird_sdk/cli/commands/build.py b/src/tinybird_sdk/cli/commands/build.py index 457a6d3..935bd43 100644 --- a/src/tinybird_sdk/cli/commands/build.py +++ b/src/tinybird_sdk/cli/commands/build.py @@ -5,7 +5,7 @@ import time from typing import Any -from ...api.branches import get_or_create_branch +from ...api.branches import CreateBranchOptions, get_or_create_branch from ...api.build import build_to_tinybird from ...api.dashboard import get_branch_dashboard_url, get_local_dashboard_url from ...api.local import ( @@ -138,9 +138,14 @@ def run_build(options: BuildCommandOptions | dict[str, Any] | None = None) -> Bu if not normalized.token_override: try: + branch_options = None + branch_value = config.get("branch_data_mode") + if branch_value and config.get("dev_mode") != "local": + branch_options = CreateBranchOptions(branch_data_mode=branch_value) branch = get_or_create_branch( {"base_url": config["base_url"], "token": config["token"]}, config["tinybird_branch"], + options=branch_options, ) if not branch.get("token"): return BuildCommandResult( diff --git a/src/tinybird_sdk/cli/commands/clear.py b/src/tinybird_sdk/cli/commands/clear.py index 176a446..c230546 100644 --- a/src/tinybird_sdk/cli/commands/clear.py +++ b/src/tinybird_sdk/cli/commands/clear.py @@ -5,7 +5,7 @@ import time from typing import Any -from ...api.branches import clear_branch +from ...api.branches import CreateBranchOptions, clear_branch from ...api.local import clear_local_workspace, get_local_tokens, get_local_workspace_name from ..config import load_config_async @@ -60,8 +60,15 @@ def run_clear(options: ClearCommandOptions | dict[str, Any] | None = None) -> Cl duration_ms=int(time.time() * 1000) - start, ) + branch_options = None + branch_value = config.get("branch_data_mode") + if branch_value and config.get("dev_mode") != "local": + branch_options = CreateBranchOptions(branch_data_mode=branch_value) + clear_branch( - {"base_url": config["base_url"], "token": config["token"]}, config["tinybird_branch"] + {"base_url": config["base_url"], "token": config["token"]}, + config["tinybird_branch"], + options=branch_options, ) return ClearResult( success=True, diff --git a/src/tinybird_sdk/cli/commands/preview.py b/src/tinybird_sdk/cli/commands/preview.py index 9a52a31..053b2b0 100644 --- a/src/tinybird_sdk/cli/commands/preview.py +++ b/src/tinybird_sdk/cli/commands/preview.py @@ -5,7 +5,7 @@ import time from typing import Any -from ...api.branches import create_branch, delete_branch, get_branch +from ...api.branches import CreateBranchOptions, create_branch, delete_branch, get_branch from ...api.build import build_to_tinybird from ...api.deploy import deploy_to_main from ...api.local import LocalNotRunningError, get_local_tokens, get_or_create_local_workspace @@ -151,8 +151,14 @@ def run_preview( except Exception: pass + branch_options = None + branch_value = config.get("branch_data_mode") + if branch_value and config.get("dev_mode") != "local": + branch_options = CreateBranchOptions(branch_data_mode=branch_value) branch = create_branch( - {"base_url": config["base_url"], "token": config["token"]}, preview_branch_name + {"base_url": config["base_url"], "token": config["token"]}, + preview_branch_name, + options=branch_options, ) except Exception as error: return PreviewCommandResult( diff --git a/src/tinybird_sdk/cli/config.py b/src/tinybird_sdk/cli/config.py index 49a0b59..990ff7b 100644 --- a/src/tinybird_sdk/cli/config.py +++ b/src/tinybird_sdk/cli/config.py @@ -8,7 +8,12 @@ from typing import Any from .config_loader import load_config_file -from .config_types import DevMode, TinybirdConfig +from .config_types import ( + BRANCH_DATA_MODE_VALUES, + BranchDataModeEnum, + DevMode, + TinybirdConfig, +) from .git import get_current_git_branch, get_tinybird_branch_name, is_main_branch DEFAULT_BASE_URL = "https://api.tinybird.co" @@ -34,6 +39,27 @@ class ResolvedConfig: tinybird_branch: str | None is_main_branch: bool dev_mode: DevMode + branch_data_mode: str | None + + +def _resolve_branch_data_mode(raw: dict[str, Any]) -> tuple[str | None, bool]: + if "branch_data_on_create" in raw: + raise ValueError("`branch_data_on_create` has been renamed to `branch_data_mode`.") + + value = raw.get("branch_data_mode") + if value is None: + return BranchDataModeEnum.LAST_PARTITION.value, False + if not isinstance(value, str): + raise ValueError("branch_data_mode must be a string.") + + mode = value.strip().lower() + if not mode: + return BranchDataModeEnum.LAST_PARTITION.value, False + if mode not in BRANCH_DATA_MODE_VALUES: + raise ValueError( + f"Invalid branch_data_mode '{value}'. Allowed values are: {', '.join(BRANCH_DATA_MODE_VALUES)}." + ) + return mode, True def load_env_files(directory: str) -> None: @@ -176,6 +202,14 @@ def _resolve_config(config: TinybirdConfig, config_path: str) -> ResolvedConfig: or DEFAULT_BASE_URL ) + branch_data_mode, branch_data_mode_explicit = _resolve_branch_data_mode(asdict(config)) + dev_mode = config.dev_mode or "branch" + if branch_data_mode_explicit and branch_data_mode and dev_mode == "local": + print( + "Warning: branch_data_mode is set in tinybird.config.json but dev_mode='local'. " + "Branch data settings only apply to cloud branches." + ) + return ResolvedConfig( include=include, token=token, @@ -185,7 +219,8 @@ def _resolve_config(config: TinybirdConfig, config_path: str) -> ResolvedConfig: git_branch=get_current_git_branch(), tinybird_branch=get_tinybird_branch_name(), is_main_branch=is_main_branch(), - dev_mode=config.dev_mode or "branch", + dev_mode=dev_mode, + branch_data_mode=branch_data_mode, ) diff --git a/src/tinybird_sdk/cli/config_types.py b/src/tinybird_sdk/cli/config_types.py index 56c02e4..8e998a2 100644 --- a/src/tinybird_sdk/cli/config_types.py +++ b/src/tinybird_sdk/cli/config_types.py @@ -1,9 +1,16 @@ from __future__ import annotations from dataclasses import dataclass, field +from enum import StrEnum from typing import Literal DevMode = Literal["branch", "local"] +BranchDataMode = Literal["last_partition"] +BRANCH_DATA_MODE_VALUES: tuple[str, ...] = ("last_partition",) + + +class BranchDataModeEnum(StrEnum): + LAST_PARTITION = "last_partition" @dataclass(frozen=True, slots=True) @@ -13,3 +20,4 @@ class TinybirdConfig: token: str | None = None base_url: str | None = None dev_mode: DevMode | None = None + branch_data_mode: str | None = None diff --git a/src/tinybird_sdk/client/base.py b/src/tinybird_sdk/client/base.py index 6f0fb36..b48ba09 100644 --- a/src/tinybird_sdk/client/base.py +++ b/src/tinybird_sdk/client/base.py @@ -4,7 +4,7 @@ from typing import Any, cast from ..api.api import TinybirdApi, TinybirdApiError -from ..api.branches import get_or_create_branch +from ..api.branches import CreateBranchOptions, get_or_create_branch from ..cli.config import load_config_async from .preview import get_preview_branch_name, is_preview_environment from .tokens import TokensNamespace @@ -159,12 +159,18 @@ def _resolve_branch_context(self) -> ClientContext: ) branch_name = config["tinybird_branch"] + branch_options = None + branch_value = config.get("branch_data_mode") + if branch_value and config.get("dev_mode") != "local": + branch_options = CreateBranchOptions(branch_data_mode=branch_value) + branch = get_or_create_branch( { "base_url": self._config["base_url"], "token": self._config["token"], }, branch_name, + options=branch_options, ) if not branch.get("token"): diff --git a/tests/test_api_branches_options.py b/tests/test_api_branches_options.py new file mode 100644 index 0000000..a04e1f0 --- /dev/null +++ b/tests/test_api_branches_options.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +from typing import Any +from urllib.parse import parse_qs, urlparse + +import pytest + +import tinybird_sdk.api.branches as branches_module +from tinybird_sdk.api.branches import CreateBranchOptions, clear_branch, create_branch + + +class _FakeResponse: + def __init__(self, status_code: int, payload: dict[str, Any]): + self.status_code = status_code + self._payload = payload + self.text = "" + + @property + def ok(self) -> bool: + return 200 <= self.status_code < 300 + + def json(self) -> dict[str, Any]: + return self._payload + + +def test_create_branch_uses_last_partition_data_query(monkeypatch: pytest.MonkeyPatch) -> None: + called_urls: list[str] = [] + + def fake_fetch(url: str, **_kwargs: Any) -> _FakeResponse: + called_urls.append(url) + if "/v1/environments?" in url: + return _FakeResponse(200, {"job": {"id": "job-1"}}) + if "/v0/jobs/" in url: + return _FakeResponse(200, {"status": "done"}) + return _FakeResponse( + 200, {"id": "b1", "name": "x", "created_at": "2024-01-01T00:00:00Z", "token": "p.test"} + ) + + monkeypatch.setattr(branches_module, "tinybird_fetch", fake_fetch) + create_branch( + {"base_url": "https://api.tinybird.co", "token": "p.test"}, + "x", + options=CreateBranchOptions(branch_data_mode="last_partition"), + ) + + parsed = urlparse(called_urls[0]) + query = parse_qs(parsed.query) + assert parsed.path == "/v1/environments" + assert query == {"name": ["x"], "data": ["last_partition"]} + + +def test_create_branch_without_options_keeps_default_query(monkeypatch: pytest.MonkeyPatch) -> None: + called_urls: list[str] = [] + + def fake_fetch(url: str, **_kwargs: Any) -> _FakeResponse: + called_urls.append(url) + if "/v1/environments?" in url: + return _FakeResponse(200, {"job": {"id": "job-1"}}) + if "/v0/jobs/" in url: + return _FakeResponse(200, {"status": "done"}) + return _FakeResponse( + 200, {"id": "b1", "name": "x", "created_at": "2024-01-01T00:00:00Z", "token": "p.test"} + ) + + monkeypatch.setattr(branches_module, "tinybird_fetch", fake_fetch) + create_branch({"base_url": "https://api.tinybird.co", "token": "p.test"}, "x") + + parsed = urlparse(called_urls[0]) + query = parse_qs(parsed.query) + assert parsed.path == "/v1/environments" + assert query == {"name": ["x"]} + assert "data" not in query + assert "ignore_datasources" not in query + + +def test_clear_branch_forwards_create_options(monkeypatch: pytest.MonkeyPatch) -> None: + captured_options: list[CreateBranchOptions | None] = [] + + monkeypatch.setattr(branches_module, "delete_branch", lambda *_args, **_kwargs: None) + + def fake_create_branch( + _config: dict[str, Any], _name: str, options: CreateBranchOptions | None = None + ) -> Any: + captured_options.append(options) + return {"id": "b1", "name": "x", "created_at": "2024-01-01T00:00:00Z", "token": "p.test"} + + monkeypatch.setattr(branches_module, "create_branch", fake_create_branch) + + clear_branch( + {"base_url": "https://api.tinybird.co", "token": "p.test"}, + "x", + options=CreateBranchOptions(branch_data_mode="last_partition"), + ) + + assert len(captured_options) == 1 + assert captured_options[0] is not None + assert captured_options[0].branch_data_mode == "last_partition" diff --git a/tests/test_cli_branch_config.py b/tests/test_cli_branch_config.py new file mode 100644 index 0000000..98e4ee0 --- /dev/null +++ b/tests/test_cli_branch_config.py @@ -0,0 +1,91 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from tinybird_sdk.cli.config import _resolve_branch_data_mode, load_config + + +def test_branch_data_mode_last_partition() -> None: + mode, explicit = _resolve_branch_data_mode({"branch_data_mode": "last_partition"}) + assert mode == "last_partition" + assert explicit is True + + +def test_branch_data_mode_missing_defaults_to_last_partition() -> None: + mode, explicit = _resolve_branch_data_mode({}) + assert mode == "last_partition" + assert explicit is False + + +def test_branch_data_mode_empty_defaults_to_last_partition() -> None: + mode, explicit = _resolve_branch_data_mode({"branch_data_mode": " "}) + assert mode == "last_partition" + assert explicit is False + + +def test_branch_data_mode_rejects_legacy_key() -> None: + with pytest.raises(ValueError, match="renamed to `branch_data_mode`"): + _resolve_branch_data_mode({"branch_data_on_create": "last_partition"}) + + +def test_branch_data_mode_rejects_all_partitions() -> None: + with pytest.raises(ValueError, match="Invalid branch_data_mode"): + _resolve_branch_data_mode({"branch_data_mode": "all_partitions"}) + + +def test_branch_data_mode_invalid_value() -> None: + with pytest.raises(ValueError, match="Invalid branch_data_mode"): + _resolve_branch_data_mode({"branch_data_mode": "invalid"}) + + +def test_branch_data_mode_non_string() -> None: + with pytest.raises(ValueError, match="must be a string"): + _resolve_branch_data_mode({"branch_data_mode": 1}) + + +def test_load_config_warns_when_local_mode_explicit_branch_data_mode( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + project = tmp_path / "project" + project.mkdir() + (project / "tinybird.config.json").write_text( + json.dumps( + { + "include": ["lib/datasources.py"], + "token": "p.test", + "base_url": "https://api.tinybird.co", + "dev_mode": "local", + "branch_data_mode": "last_partition", + } + ), + encoding="utf-8", + ) + + load_config(str(project)) + captured = capsys.readouterr() + assert "branch_data_mode is set" in captured.out + + +def test_load_config_does_not_warn_when_branch_data_mode_is_implicit( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + project = tmp_path / "project" + project.mkdir() + (project / "tinybird.config.json").write_text( + json.dumps( + { + "include": ["lib/datasources.py"], + "token": "p.test", + "base_url": "https://api.tinybird.co", + "dev_mode": "local", + } + ), + encoding="utf-8", + ) + + load_config(str(project)) + captured = capsys.readouterr() + assert "branch_data_mode is set" not in captured.out diff --git a/tests/test_client_parity.py b/tests/test_client_parity.py index 434626f..70b132f 100644 --- a/tests/test_client_parity.py +++ b/tests/test_client_parity.py @@ -43,6 +43,8 @@ def query( def test_client_branch_context_uses_branch_token(monkeypatch: pytest.MonkeyPatch) -> None: + captured: dict[str, Any] = {} + monkeypatch.setattr(client_base, "is_preview_environment", lambda: False) monkeypatch.setattr( client_base, @@ -51,13 +53,16 @@ def test_client_branch_context_uses_branch_token(monkeypatch: pytest.MonkeyPatch "git_branch": "feature/alpha", "tinybird_branch": "feature_alpha", "is_main_branch": False, + "dev_mode": "branch", + "branch_data_mode": "last_partition", }, ) - monkeypatch.setattr( - client_base, - "get_or_create_branch", - lambda *_args, **_kwargs: {"token": "branch_token"}, - ) + + def fake_get_or_create_branch(*_args: Any, **kwargs: Any) -> dict[str, Any]: + captured.update(kwargs) + return {"token": "branch_token"} + + monkeypatch.setattr(client_base, "get_or_create_branch", fake_get_or_create_branch) client = TinybirdClient( { @@ -71,6 +76,7 @@ def test_client_branch_context_uses_branch_token(monkeypatch: pytest.MonkeyPatch assert context["token"] == "branch_token" assert context["is_branch_token"] is True assert context["branch_name"] == "feature_alpha" + assert captured["options"].branch_data_mode == "last_partition" def test_tokens_namespace_wraps_token_api_errors(monkeypatch: pytest.MonkeyPatch) -> None: diff --git a/uv.lock b/uv.lock index 9b5f8a4..5d81094 100644 --- a/uv.lock +++ b/uv.lock @@ -1190,7 +1190,7 @@ wheels = [ [[package]] name = "tinybird-sdk" -version = "0.2.0" +version = "0.3.0" source = { editable = "." } dependencies = [ { name = "tinybird" }, @@ -1205,7 +1205,7 @@ dev = [ ] [package.metadata] -requires-dist = [{ name = "tinybird", specifier = "==4.6.0" }] +requires-dist = [{ name = "tinybird", specifier = ">=4.6.0,<4.7.0" }] [package.metadata.requires-dev] dev = [