diff --git a/pyproject.toml b/pyproject.toml index 18de6422..555b19bd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,6 +80,8 @@ sandbox-wasm = ["wasmtime>=25"] dashboard = ["textual>=0.80"] speech = ["faster-whisper>=1.0"] speech-deepgram = ["deepgram-sdk>=3.0"] +eval-wandb = ["wandb>=0.17"] +eval-sheets = ["gspread>=6.0", "google-auth>=2.0"] docs = [ "mkdocs>=1.6", "mkdocs-material>=9.5", diff --git a/src/openjarvis/cli/eval_cmd.py b/src/openjarvis/cli/eval_cmd.py index ce5d116b..3a9dc8dc 100644 --- a/src/openjarvis/cli/eval_cmd.py +++ b/src/openjarvis/cli/eval_cmd.py @@ -113,6 +113,34 @@ def eval_list() -> None: "-o", "--output", "output_path", default=None, type=click.Path(), help="Output JSONL path.", ) +@click.option( + "--wandb-project", "wandb_project", default="", + help="W&B project name (enables W&B tracking).", +) +@click.option( + "--wandb-entity", "wandb_entity", default="", + help="W&B entity (team or user).", +) +@click.option( + "--wandb-tags", "wandb_tags", default="", + help="Comma-separated W&B tags.", +) +@click.option( + "--wandb-group", "wandb_group", default="", + help="W&B run group.", +) +@click.option( + "--sheets-id", "sheets_spreadsheet_id", default="", + help="Google Sheets spreadsheet ID.", +) +@click.option( + "--sheets-worksheet", "sheets_worksheet", default="Results", + help="Google Sheets worksheet name.", +) +@click.option( + "--sheets-creds", "sheets_credentials_path", default="", + help="Path to Google service account JSON.", +) @click.option( "-v", "--verbose", "verbose", is_flag=True, default=False, help="Verbose logging.", @@ -127,6 +155,13 @@ def eval_run( tools: str, telemetry: bool, output_path: Optional[str], + wandb_project: str, + wandb_entity: str, + wandb_tags: str, + wandb_group: str, + sheets_spreadsheet_id: str, + sheets_worksheet: str, + sheets_credentials_path: str, verbose: bool, ) -> None: """Run evaluation benchmarks.""" @@ -216,6 +251,13 @@ def eval_run( tools=tool_list, output_path=output_path, telemetry=telemetry, + wandb_project=wandb_project, + wandb_entity=wandb_entity, + wandb_tags=wandb_tags, + wandb_group=wandb_group, + sheets_spreadsheet_id=sheets_spreadsheet_id, + sheets_worksheet=sheets_worksheet, + sheets_credentials_path=sheets_credentials_path, ) try: diff --git a/src/openjarvis/evals/cli.py b/src/openjarvis/evals/cli.py index 1318bc04..7395bfd1 100644 --- a/src/openjarvis/evals/cli.py +++ b/src/openjarvis/evals/cli.py @@ -217,6 +217,39 @@ def _print_summary( print_completion(console, summary, output_path, traces_dir) +def _build_trackers(config) -> list: + """Build tracker instances from RunConfig fields.""" + trackers = [] + if getattr(config, "wandb_project", ""): + try: + from openjarvis.evals.trackers.wandb_tracker import WandbTracker + trackers.append(WandbTracker( + project=config.wandb_project, + entity=getattr(config, "wandb_entity", ""), + tags=getattr(config, "wandb_tags", ""), + group=getattr(config, "wandb_group", ""), + )) + except ImportError as exc: + raise click.ClickException( + f"wandb not installed: {exc}\n" + "Install with: pip install 'openjarvis[eval-wandb]'" + ) from exc + if getattr(config, "sheets_spreadsheet_id", ""): + try: + from openjarvis.evals.trackers.sheets_tracker import SheetsTracker + trackers.append(SheetsTracker( + spreadsheet_id=config.sheets_spreadsheet_id, + worksheet=getattr(config, "sheets_worksheet", "Results"), + credentials_path=getattr(config, "sheets_credentials_path", ""), + )) + except ImportError as exc: + raise click.ClickException( + f"gspread not installed: {exc}\n" + "Install with: pip install 'openjarvis[eval-sheets]'" + ) from exc + return trackers + + def _run_single(config, console: Optional[Console] = None) -> object: """Run a single eval from a RunConfig and return the summary.""" from openjarvis.evals.core.runner import EvalRunner @@ -236,7 +269,8 @@ def _run_single(config, console: Optional[Console] = None) -> object: judge_backend = _build_judge_backend(config.judge_model) scorer = _build_scorer(config.benchmark, judge_backend, config.judge_model) - runner = EvalRunner(config, dataset, eval_backend, scorer) + trackers = _build_trackers(config) + runner = EvalRunner(config, dataset, eval_backend, scorer, trackers=trackers) try: num_samples = config.max_samples or 0 # Use progress bar if we know the sample count @@ -292,6 +326,11 @@ def _run_from_config(config_path: str, verbose: bool) -> None: output_dir = Path(suite.run.output_dir) output_dir.mkdir(parents=True, exist_ok=True) + # Auto-set wandb_group to suite name if W&B enabled and no explicit group + for rc in run_configs: + if rc.wandb_project and not rc.wandb_group: + rc.wandb_group = suite_name + summaries = [] for i, rc in enumerate(run_configs, 1): print_section( @@ -355,12 +394,30 @@ def main(): help="Enable GPU metrics collection") @click.option("--compact", is_flag=True, default=False, help="Dense single-table output") @click.option("--trace-detail", is_flag=True, default=False, help="Full per-step trace listing") +@click.option("--wandb-project", default="", + help="W&B project name (enables tracking)") +@click.option("--wandb-entity", default="", + help="W&B entity (team or user)") +@click.option("--wandb-tags", default="", + help="Comma-separated W&B tags") +@click.option("--wandb-group", default="", + help="W&B run group") +@click.option("--sheets-id", "sheets_spreadsheet_id", default="", + help="Google Sheets spreadsheet ID") +@click.option("--sheets-worksheet", default="Results", + help="Google Sheets worksheet name") +@click.option("--sheets-creds", "sheets_credentials_path", + default="", + help="Service account JSON path") @click.option("-v", "--verbose", is_flag=True, help="Verbose logging") @click.pass_context def run(ctx, config_path, benchmark, backend, model, engine_key, agent_name, tools, max_samples, max_workers, judge_model, output_path, seed, dataset_split, temperature, max_tokens, telemetry, gpu_metrics, - compact, trace_detail, verbose): + compact, trace_detail, + wandb_project, wandb_entity, wandb_tags, wandb_group, + sheets_spreadsheet_id, sheets_worksheet, sheets_credentials_path, + verbose): """Run a single benchmark evaluation, or a full suite from a TOML config.""" _setup_logging(verbose) @@ -404,6 +461,13 @@ def run(ctx, config_path, benchmark, backend, model, engine_key, agent_name, dataset_split=dataset_split, telemetry=telemetry, gpu_metrics=gpu_metrics, + wandb_project=wandb_project, + wandb_entity=wandb_entity, + wandb_tags=wandb_tags, + wandb_group=wandb_group, + sheets_spreadsheet_id=sheets_spreadsheet_id, + sheets_worksheet=sheets_worksheet, + sheets_credentials_path=sheets_credentials_path, ) # Banner + config @@ -493,7 +557,8 @@ def run_all(model, engine_key, max_samples, max_workers, judge_model, judge_backend = _build_judge_backend(judge_model) scorer = _build_scorer(bench_name, judge_backend, judge_model) - runner = EvalRunner(config, dataset, eval_backend, scorer) + trackers = _build_trackers(config) + runner = EvalRunner(config, dataset, eval_backend, scorer, trackers=trackers) try: if max_samples and max_samples > 0: with Progress( diff --git a/src/openjarvis/evals/core/config.py b/src/openjarvis/evals/core/config.py index 966d4fdc..af09a9de 100644 --- a/src/openjarvis/evals/core/config.py +++ b/src/openjarvis/evals/core/config.py @@ -99,6 +99,13 @@ def load_eval_config(path: str | Path) -> EvalSuiteConfig: gpu_metrics=bool(run_raw.get("gpu_metrics", False)), warmup_samples=int(run_raw.get("warmup_samples", 0)), energy_vendor=run_raw.get("energy_vendor", ""), + wandb_project=run_raw.get("wandb_project", ""), + wandb_entity=run_raw.get("wandb_entity", ""), + wandb_tags=run_raw.get("wandb_tags", ""), + wandb_group=run_raw.get("wandb_group", ""), + sheets_spreadsheet_id=run_raw.get("sheets_spreadsheet_id", ""), + sheets_worksheet=run_raw.get("sheets_worksheet", "Results"), + sheets_credentials_path=run_raw.get("sheets_credentials_path", ""), ) # Parse [[models]] @@ -243,6 +250,13 @@ def expand_suite(suite: EvalSuiteConfig) -> List[RunConfig]: gpu_metrics=suite.run.gpu_metrics, metadata=model_meta, warmup_samples=suite.run.warmup_samples, + wandb_project=suite.run.wandb_project, + wandb_entity=suite.run.wandb_entity, + wandb_tags=suite.run.wandb_tags, + wandb_group=suite.run.wandb_group, + sheets_spreadsheet_id=suite.run.sheets_spreadsheet_id, + sheets_worksheet=suite.run.sheets_worksheet, + sheets_credentials_path=suite.run.sheets_credentials_path, )) return configs diff --git a/src/openjarvis/evals/core/runner.py b/src/openjarvis/evals/core/runner.py index 07a9a0eb..66ec57f1 100644 --- a/src/openjarvis/evals/core/runner.py +++ b/src/openjarvis/evals/core/runner.py @@ -14,7 +14,14 @@ from typing import Any, Callable, Dict, List, Optional from openjarvis.evals.core.backend import InferenceBackend from openjarvis.evals.core.dataset import DatasetProvider from openjarvis.evals.core.scorer import Scorer -from openjarvis.evals.core.types import EvalRecord, EvalResult, MetricStats, RunConfig, RunSummary +from openjarvis.evals.core.tracker import ResultTracker +from openjarvis.evals.core.types import ( + EvalRecord, + EvalResult, + MetricStats, + RunConfig, + RunSummary, +) try: from openjarvis.telemetry.efficiency import compute_efficiency @@ -33,11 +40,13 @@ class EvalRunner: dataset: DatasetProvider, backend: InferenceBackend, scorer: Scorer, + trackers: Optional[List[ResultTracker]] = None, ) -> None: self._config = config self._dataset = dataset self._backend = backend self._scorer = scorer + self._trackers: List[ResultTracker] = trackers or [] self._results: List[EvalResult] = [] self._output_file: Optional[Any] = None @@ -79,6 +88,16 @@ class EvalRunner: output_path.parent.mkdir(parents=True, exist_ok=True) self._output_file = open(output_path, "w") + # Notify trackers of run start + for tracker in self._trackers: + try: + tracker.on_run_start(cfg) + except Exception as exc: + LOGGER.warning( + "Tracker %s.on_run_start failed: %s", + type(tracker).__name__, exc, + ) + total = len(records) try: with ThreadPoolExecutor(max_workers=cfg.max_workers) as pool: @@ -99,6 +118,23 @@ class EvalRunner: ended_at = time.time() summary = self._compute_summary(records, started_at, ended_at) + # Notify trackers of summary and run end + for tracker in self._trackers: + try: + tracker.on_summary(summary) + except Exception as exc: + LOGGER.warning( + "Tracker %s.on_summary failed: %s", + type(tracker).__name__, exc, + ) + try: + tracker.on_run_end() + except Exception as exc: + LOGGER.warning( + "Tracker %s.on_run_end failed: %s", + type(tracker).__name__, exc, + ) + # Write summary JSON alongside JSONL traces_dir: Optional[Path] = None if output_path: @@ -255,6 +291,16 @@ class EvalRunner: self._output_file.write(json.dumps(record_dict) + "\n") self._output_file.flush() + # Notify trackers of each result + for tracker in self._trackers: + try: + tracker.on_result(result, self._config) + except Exception as exc: + LOGGER.warning( + "Tracker %s.on_result failed: %s", + type(tracker).__name__, exc, + ) + def _resolve_output_path(self) -> Optional[Path]: """Determine the output file path.""" if self._config.output_path: diff --git a/src/openjarvis/evals/core/tracker.py b/src/openjarvis/evals/core/tracker.py new file mode 100644 index 00000000..83b62deb --- /dev/null +++ b/src/openjarvis/evals/core/tracker.py @@ -0,0 +1,34 @@ +"""ResultTracker ABC for external experiment tracking.""" + +from __future__ import annotations + +from abc import ABC, abstractmethod + +from openjarvis.evals.core.types import EvalResult, RunConfig, RunSummary + + +class ResultTracker(ABC): + """Abstract base class for experiment result trackers. + + Lifecycle: on_run_start -> on_result (per sample) + -> on_summary -> on_run_end. + """ + + @abstractmethod + def on_run_start(self, config: RunConfig) -> None: + """Called once before evaluation begins.""" + + @abstractmethod + def on_result(self, result: EvalResult, config: RunConfig) -> None: + """Called after each sample is evaluated.""" + + @abstractmethod + def on_summary(self, summary: RunSummary) -> None: + """Called after all samples are evaluated with aggregate stats.""" + + @abstractmethod + def on_run_end(self) -> None: + """Called at the very end of a run for cleanup.""" + + +__all__ = ["ResultTracker"] diff --git a/src/openjarvis/evals/core/types.py b/src/openjarvis/evals/core/types.py index 22702dd0..6f4c10ff 100644 --- a/src/openjarvis/evals/core/types.py +++ b/src/openjarvis/evals/core/types.py @@ -71,6 +71,13 @@ class RunConfig: gpu_metrics: bool = False metadata: Dict[str, Any] = field(default_factory=dict) warmup_samples: int = 0 + wandb_project: str = "" + wandb_entity: str = "" + wandb_tags: str = "" + wandb_group: str = "" + sheets_spreadsheet_id: str = "" + sheets_worksheet: str = "Results" + sheets_credentials_path: str = "" @dataclass(slots=True) @@ -176,6 +183,13 @@ class ExecutionConfig: gpu_metrics: bool = False warmup_samples: int = 0 energy_vendor: str = "" + wandb_project: str = "" + wandb_entity: str = "" + wandb_tags: str = "" + wandb_group: str = "" + sheets_spreadsheet_id: str = "" + sheets_worksheet: str = "Results" + sheets_credentials_path: str = "" @dataclass(slots=True) diff --git a/src/openjarvis/evals/trackers/__init__.py b/src/openjarvis/evals/trackers/__init__.py new file mode 100644 index 00000000..61306ba6 --- /dev/null +++ b/src/openjarvis/evals/trackers/__init__.py @@ -0,0 +1,23 @@ +"""External experiment trackers for the eval framework. + +Trackers are lazily imported to avoid mandatory dependencies on wandb/gspread. +""" + +from __future__ import annotations + + +def WandbTracker(*args, **kwargs): # noqa: N802 + """Lazy constructor — imports the real class on first use.""" + from openjarvis.evals.trackers.wandb_tracker import WandbTracker as _Cls + + return _Cls(*args, **kwargs) + + +def SheetsTracker(*args, **kwargs): # noqa: N802 + """Lazy constructor — imports the real class on first use.""" + from openjarvis.evals.trackers.sheets_tracker import SheetsTracker as _Cls + + return _Cls(*args, **kwargs) + + +__all__ = ["WandbTracker", "SheetsTracker"] diff --git a/src/openjarvis/evals/trackers/sheets_tracker.py b/src/openjarvis/evals/trackers/sheets_tracker.py new file mode 100644 index 00000000..85c9e37b --- /dev/null +++ b/src/openjarvis/evals/trackers/sheets_tracker.py @@ -0,0 +1,162 @@ +"""Google Sheets experiment tracker for the eval framework.""" + +from __future__ import annotations + +import logging +import time +from typing import Any, List, Optional + +from openjarvis.evals.core.tracker import ResultTracker +from openjarvis.evals.core.types import EvalResult, MetricStats, RunConfig, RunSummary + +try: + import gspread + from google.oauth2.service_account import Credentials +except ImportError: + gspread = None # type: ignore[assignment] + Credentials = None # type: ignore[assignment,misc] + +LOGGER = logging.getLogger(__name__) + +# Canonical column order for the summary row. +SHEET_COLUMNS: List[str] = [ + "timestamp", + "benchmark", + "model", + "backend", + "total_samples", + "scored_samples", + "correct", + "accuracy", + "errors", + "mean_latency_seconds", + "total_cost_usd", + "total_energy_joules", + "avg_power_watts", + "total_input_tokens", + "total_output_tokens", + "latency_mean", + "latency_p90", + "latency_p95", + "energy_mean", + "energy_p90", + "throughput_mean", + "throughput_p90", + "ipw_mean", + "ipj_mean", + "mfu_mean", + "mbu_mean", + "ttft_mean", + "ttft_p90", + "gpu_utilization_mean", +] + + +def _stat_val(ms: Optional[MetricStats], attr: str) -> Any: + """Safely extract a stat value from a MetricStats, returning '' if None.""" + if ms is None: + return "" + return getattr(ms, attr, "") + + +class SheetsTracker(ResultTracker): + """Appends a summary row to a Google Sheet after each eval run.""" + + def __init__( + self, + spreadsheet_id: str, + worksheet: str = "Results", + credentials_path: str = "", + ) -> None: + if gspread is None: + raise ImportError( + "gspread is not installed. " + "Install it with: pip install 'openjarvis[eval-sheets]'" + ) + self._spreadsheet_id = spreadsheet_id + self._worksheet_name = worksheet + self._credentials_path = credentials_path + + def on_run_start(self, config: RunConfig) -> None: + pass + + def on_result(self, result: EvalResult, config: RunConfig) -> None: + # No-op: summary-only to avoid excessive API calls. + pass + + def on_summary(self, summary: RunSummary) -> None: + row = self._build_row(summary) + try: + gc = self._authorize() + spreadsheet = gc.open_by_key(self._spreadsheet_id) + try: + ws = spreadsheet.worksheet(self._worksheet_name) + except gspread.exceptions.WorksheetNotFound: + ws = spreadsheet.add_worksheet( + title=self._worksheet_name, rows=1000, cols=len(SHEET_COLUMNS), + ) + # Ensure header row exists (idempotent) + existing = ws.row_values(1) + if not existing or existing[0] != SHEET_COLUMNS[0]: + ws.update(range_name="A1", values=[SHEET_COLUMNS]) + ws.append_row(row, value_input_option="RAW") + LOGGER.info("Appended summary row to Google Sheet") + except Exception as exc: + LOGGER.warning("SheetsTracker.on_summary failed: %s", exc) + + def on_run_end(self) -> None: + pass + + def _authorize(self): + """Authenticate with Google Sheets API.""" + scopes = [ + "https://www.googleapis.com/auth/spreadsheets", + "https://www.googleapis.com/auth/drive", + ] + if self._credentials_path: + creds = Credentials.from_service_account_file( + self._credentials_path, scopes=scopes, + ) + else: + # Fall back to Application Default Credentials + import google.auth + + creds, _ = google.auth.default(scopes=scopes) + return gspread.authorize(creds) + + def _build_row(self, s: RunSummary) -> List[Any]: + """Build a flat row matching SHEET_COLUMNS order.""" + return [ + time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + s.benchmark, + s.model, + s.backend, + s.total_samples, + s.scored_samples, + s.correct, + s.accuracy, + s.errors, + s.mean_latency_seconds, + s.total_cost_usd, + s.total_energy_joules, + s.avg_power_watts, + s.total_input_tokens, + s.total_output_tokens, + _stat_val(s.latency_stats, "mean"), + _stat_val(s.latency_stats, "p90"), + _stat_val(s.latency_stats, "p95"), + _stat_val(s.energy_stats, "mean"), + _stat_val(s.energy_stats, "p90"), + _stat_val(s.throughput_stats, "mean"), + _stat_val(s.throughput_stats, "p90"), + _stat_val(s.ipw_stats, "mean"), + _stat_val(s.ipj_stats, "mean"), + _stat_val(s.mfu_stats, "mean"), + _stat_val(s.mbu_stats, "mean"), + _stat_val(s.ttft_stats, "mean"), + _stat_val(s.ttft_stats, "p90"), + _stat_val(s.gpu_utilization_stats, "mean"), + ] + + +__all__ = ["SheetsTracker", "SHEET_COLUMNS"] diff --git a/src/openjarvis/evals/trackers/wandb_tracker.py b/src/openjarvis/evals/trackers/wandb_tracker.py new file mode 100644 index 00000000..d613f03d --- /dev/null +++ b/src/openjarvis/evals/trackers/wandb_tracker.py @@ -0,0 +1,154 @@ +"""W&B experiment tracker for the eval framework.""" + +from __future__ import annotations + +import logging +from typing import Any, Dict, List, Optional + +from openjarvis.evals.core.tracker import ResultTracker +from openjarvis.evals.core.types import EvalResult, MetricStats, RunConfig, RunSummary + +try: + import wandb +except ImportError: + wandb = None # type: ignore[assignment] + +LOGGER = logging.getLogger(__name__) + + +def _flatten_metric_stats(prefix: str, ms: Optional[MetricStats]) -> Dict[str, float]: + """Flatten a MetricStats into a dict with prefixed keys.""" + if ms is None: + return {} + return { + f"{prefix}_mean": ms.mean, + f"{prefix}_median": ms.median, + f"{prefix}_min": ms.min, + f"{prefix}_max": ms.max, + f"{prefix}_std": ms.std, + f"{prefix}_p90": ms.p90, + f"{prefix}_p95": ms.p95, + f"{prefix}_p99": ms.p99, + } + + +class WandbTracker(ResultTracker): + """Streams per-sample metrics to Weights & Biases.""" + + def __init__( + self, + project: str, + entity: str = "", + tags: str = "", + group: str = "", + ) -> None: + if wandb is None: + raise ImportError( + "wandb is not installed. " + "Install it with: pip install 'openjarvis[eval-wandb]'" + ) + self._project = project + self._entity = entity or None + self._tags: List[str] = [ + t.strip() for t in tags.split(",") if t.strip() + ] if tags else [] + self._group = group or None + self._run: Any = None + self._step = 0 + + def on_run_start(self, config: RunConfig) -> None: + run_config = { + "benchmark": config.benchmark, + "model": config.model, + "backend": config.backend, + "max_samples": config.max_samples, + "max_workers": config.max_workers, + "temperature": config.temperature, + "max_tokens": config.max_tokens, + "seed": config.seed, + } + if config.agent_name: + run_config["agent_name"] = config.agent_name + if config.tools: + run_config["tools"] = ",".join(config.tools) + if config.engine_key: + run_config["engine_key"] = config.engine_key + + self._run = wandb.init( + project=self._project, + entity=self._entity, + tags=self._tags or None, + group=self._group, + config=run_config, + reinit=True, + ) + self._step = 0 + + def on_result(self, result: EvalResult, config: RunConfig) -> None: + if self._run is None: + return + self._step += 1 + log_data: Dict[str, Any] = { + "sample/is_correct": 1.0 if result.is_correct else 0.0, + "sample/latency_seconds": result.latency_seconds, + "sample/prompt_tokens": result.prompt_tokens, + "sample/completion_tokens": result.completion_tokens, + "sample/cost_usd": result.cost_usd, + "sample/ttft": result.ttft, + "sample/energy_joules": result.energy_joules, + "sample/power_watts": result.power_watts, + "sample/throughput_tok_per_sec": result.throughput_tok_per_sec, + "sample/ipw": result.ipw, + "sample/ipj": result.ipj, + } + if result.error: + log_data["sample/has_error"] = 1.0 + wandb.log(log_data, step=self._step) + + def on_summary(self, summary: RunSummary) -> None: + if self._run is None: + return + flat: Dict[str, Any] = { + "accuracy": summary.accuracy, + "total_samples": summary.total_samples, + "scored_samples": summary.scored_samples, + "correct": summary.correct, + "errors": summary.errors, + "mean_latency_seconds": summary.mean_latency_seconds, + "total_cost_usd": summary.total_cost_usd, + "total_energy_joules": summary.total_energy_joules, + "avg_power_watts": summary.avg_power_watts, + "total_input_tokens": summary.total_input_tokens, + "total_output_tokens": summary.total_output_tokens, + } + flat.update(_flatten_metric_stats("accuracy", summary.accuracy_stats)) + flat.update(_flatten_metric_stats("latency", summary.latency_stats)) + flat.update(_flatten_metric_stats("ttft", summary.ttft_stats)) + flat.update(_flatten_metric_stats("energy", summary.energy_stats)) + flat.update(_flatten_metric_stats("power", summary.power_stats)) + flat.update( + _flatten_metric_stats("gpu_utilization", summary.gpu_utilization_stats) + ) + flat.update(_flatten_metric_stats("throughput", summary.throughput_stats)) + flat.update(_flatten_metric_stats("mfu", summary.mfu_stats)) + flat.update(_flatten_metric_stats("mbu", summary.mbu_stats)) + flat.update(_flatten_metric_stats("ipw", summary.ipw_stats)) + flat.update(_flatten_metric_stats("ipj", summary.ipj_stats)) + flat.update(_flatten_metric_stats( + "energy_per_output_token", + summary.energy_per_output_token_stats, + )) + flat.update(_flatten_metric_stats( + "throughput_per_watt", + summary.throughput_per_watt_stats, + )) + flat.update(_flatten_metric_stats("itl", summary.itl_stats)) + wandb.run.summary.update(flat) + + def on_run_end(self) -> None: + if self._run is not None: + self._run.finish() + self._run = None + + +__all__ = ["WandbTracker"] diff --git a/tests/evals/test_trackers.py b/tests/evals/test_trackers.py new file mode 100644 index 00000000..9481fade --- /dev/null +++ b/tests/evals/test_trackers.py @@ -0,0 +1,333 @@ +"""Tests for eval result trackers (W&B + Google Sheets).""" + +from __future__ import annotations + +import sys +from typing import List +from unittest.mock import MagicMock, patch + +import pytest + +from openjarvis.evals.core.tracker import ResultTracker +from openjarvis.evals.core.types import EvalResult, RunConfig, RunSummary + +# --------------------------------------------------------------------------- +# Test double +# --------------------------------------------------------------------------- + +class RecordingTracker(ResultTracker): + """Records all lifecycle calls for testing.""" + + def __init__(self) -> None: + self.calls: List[str] = [] + self.results: List[EvalResult] = [] + self.summary: RunSummary | None = None + + def on_run_start(self, config: RunConfig) -> None: + self.calls.append("on_run_start") + + def on_result(self, result: EvalResult, config: RunConfig) -> None: + self.calls.append("on_result") + self.results.append(result) + + def on_summary(self, summary: RunSummary) -> None: + self.calls.append("on_summary") + self.summary = summary + + def on_run_end(self) -> None: + self.calls.append("on_run_end") + + +class CrashingTracker(ResultTracker): + """Raises on every lifecycle call.""" + + def on_run_start(self, config: RunConfig) -> None: + raise RuntimeError("boom start") + + def on_result(self, result: EvalResult, config: RunConfig) -> None: + raise RuntimeError("boom result") + + def on_summary(self, summary: RunSummary) -> None: + raise RuntimeError("boom summary") + + def on_run_end(self) -> None: + raise RuntimeError("boom end") + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _make_config(**overrides) -> RunConfig: + defaults = dict(benchmark="test", backend="jarvis-direct", model="test-model") + defaults.update(overrides) + return RunConfig(**defaults) + + +def _make_summary(**overrides) -> RunSummary: + defaults = dict( + benchmark="test", + category="chat", + backend="jarvis-direct", + model="test-model", + total_samples=10, + scored_samples=10, + correct=8, + accuracy=0.8, + errors=0, + mean_latency_seconds=1.0, + total_cost_usd=0.01, + ) + defaults.update(overrides) + return RunSummary(**defaults) + + +def _make_result(**overrides) -> EvalResult: + defaults = dict(record_id="r1", model_answer="answer", is_correct=True) + defaults.update(overrides) + return EvalResult(**defaults) + + +# --------------------------------------------------------------------------- +# RecordingTracker through EvalRunner lifecycle +# --------------------------------------------------------------------------- + +class TestRecordingTrackerIntegration: + """Test that trackers receive all lifecycle calls through EvalRunner.""" + + def test_tracker_lifecycle(self, tmp_path): + """RecordingTracker receives start, result, summary, end calls.""" + from openjarvis.evals.core.runner import EvalRunner + + # Minimal stubs + dataset = MagicMock() + record = MagicMock() + record.record_id = "r1" + record.problem = "What is 1+1?" + record.reference = "2" + record.category = "chat" + record.subject = "" + dataset.load = MagicMock() + dataset.iter_records = MagicMock(return_value=[record]) + + backend = MagicMock() + backend.generate_full = MagicMock(return_value={ + "content": "2", + "usage": {"prompt_tokens": 10, "completion_tokens": 5}, + "latency_seconds": 0.5, + }) + + scorer = MagicMock() + scorer.score = MagicMock(return_value=(True, {})) + + tracker = RecordingTracker() + config = _make_config(output_path=str(tmp_path / "out.jsonl")) + + runner = EvalRunner(config, dataset, backend, scorer, trackers=[tracker]) + runner.run() + + assert "on_run_start" in tracker.calls + assert "on_result" in tracker.calls + assert "on_summary" in tracker.calls + assert "on_run_end" in tracker.calls + # Order matters + assert tracker.calls.index("on_run_start") < tracker.calls.index("on_result") + assert tracker.calls.index("on_result") < tracker.calls.index("on_summary") + assert tracker.calls.index("on_summary") < tracker.calls.index("on_run_end") + assert len(tracker.results) == 1 + assert tracker.summary is not None + + def test_crashing_tracker_does_not_abort(self, tmp_path): + """A tracker that raises exceptions must not prevent JSONL output.""" + from openjarvis.evals.core.runner import EvalRunner + + dataset = MagicMock() + record = MagicMock() + record.record_id = "r1" + record.problem = "What?" + record.reference = "yes" + record.category = "chat" + record.subject = "" + dataset.load = MagicMock() + dataset.iter_records = MagicMock(return_value=[record]) + + backend = MagicMock() + backend.generate_full = MagicMock(return_value={ + "content": "yes", + "usage": {}, + "latency_seconds": 0.1, + }) + + scorer = MagicMock() + scorer.score = MagicMock(return_value=(True, {})) + + output = tmp_path / "out.jsonl" + config = _make_config(output_path=str(output)) + + crasher = CrashingTracker() + runner = EvalRunner(config, dataset, backend, scorer, trackers=[crasher]) + summary = runner.run() + + # Run completed, JSONL written despite crashing tracker + assert summary.total_samples == 1 + assert output.exists() + assert output.read_text().strip() != "" + + +# --------------------------------------------------------------------------- +# WandbTracker unit tests +# --------------------------------------------------------------------------- + +class TestWandbTracker: + """Unit tests for WandbTracker (mocked wandb module).""" + + def test_import_error_when_wandb_missing(self): + """WandbTracker raises ImportError when wandb is not installed.""" + with patch.dict(sys.modules, {"wandb": None}): + import openjarvis.evals.trackers.wandb_tracker as wt_mod + original = wt_mod.wandb + wt_mod.wandb = None + try: + with pytest.raises(ImportError, match="wandb is not installed"): + wt_mod.WandbTracker(project="test") + finally: + wt_mod.wandb = original + + def test_on_result_calls_wandb_log(self): + """on_result calls wandb.log with sample/ prefixed keys.""" + import openjarvis.evals.trackers.wandb_tracker as wt_mod + + mock_wandb = MagicMock() + mock_run = MagicMock() + mock_wandb.init = MagicMock(return_value=mock_run) + original = wt_mod.wandb + wt_mod.wandb = mock_wandb + try: + tracker = wt_mod.WandbTracker(project="test-proj") + config = _make_config() + tracker.on_run_start(config) + + result = _make_result(latency_seconds=0.5, energy_joules=1.0) + tracker.on_result(result, config) + + mock_wandb.log.assert_called_once() + call_args = mock_wandb.log.call_args + log_data = call_args[0][0] + assert "sample/is_correct" in log_data + assert "sample/latency_seconds" in log_data + assert log_data["sample/is_correct"] == 1.0 + assert call_args[1]["step"] == 1 + + tracker.on_run_end() + finally: + wt_mod.wandb = original + + def test_on_summary_updates_run_summary(self): + """on_summary calls wandb.run.summary.update with flat dict.""" + import openjarvis.evals.trackers.wandb_tracker as wt_mod + + mock_wandb = MagicMock() + mock_run = MagicMock() + mock_wandb.init = MagicMock(return_value=mock_run) + mock_wandb.run = mock_run + original = wt_mod.wandb + wt_mod.wandb = mock_wandb + try: + tracker = wt_mod.WandbTracker(project="test-proj") + config = _make_config() + tracker.on_run_start(config) + + summary = _make_summary() + tracker.on_summary(summary) + + mock_run.summary.update.assert_called_once() + flat = mock_run.summary.update.call_args[0][0] + assert flat["accuracy"] == 0.8 + assert flat["total_samples"] == 10 + + tracker.on_run_end() + finally: + wt_mod.wandb = original + + def test_reinit_true_for_suite_mode(self): + """wandb.init is called with reinit=True.""" + import openjarvis.evals.trackers.wandb_tracker as wt_mod + + mock_wandb = MagicMock() + mock_run = MagicMock() + mock_wandb.init = MagicMock(return_value=mock_run) + original = wt_mod.wandb + wt_mod.wandb = mock_wandb + try: + tracker = wt_mod.WandbTracker(project="test-proj", entity="team") + config = _make_config() + tracker.on_run_start(config) + + call_kwargs = mock_wandb.init.call_args[1] + assert call_kwargs["reinit"] is True + assert call_kwargs["project"] == "test-proj" + assert call_kwargs["entity"] == "team" + + tracker.on_run_end() + finally: + wt_mod.wandb = original + + +# --------------------------------------------------------------------------- +# SheetsTracker unit tests +# --------------------------------------------------------------------------- + +class TestSheetsTracker: + """Unit tests for SheetsTracker.""" + + def test_import_error_when_gspread_missing(self): + """SheetsTracker raises ImportError when gspread not installed.""" + import openjarvis.evals.trackers.sheets_tracker as st_mod + original = st_mod.gspread + st_mod.gspread = None + try: + with pytest.raises(ImportError, match="gspread is not installed"): + st_mod.SheetsTracker(spreadsheet_id="abc123") + finally: + st_mod.gspread = original + + def test_on_result_is_noop(self): + """on_result does nothing (no API calls for individual samples).""" + import openjarvis.evals.trackers.sheets_tracker as st_mod + + mock_gspread = MagicMock() + original = st_mod.gspread + st_mod.gspread = mock_gspread + original_creds = st_mod.Credentials + st_mod.Credentials = MagicMock() + try: + tracker = st_mod.SheetsTracker(spreadsheet_id="abc123") + result = _make_result() + config = _make_config() + + # on_result should not call any external API + tracker.on_result(result, config) + mock_gspread.authorize.assert_not_called() + finally: + st_mod.gspread = original + st_mod.Credentials = original_creds + + def test_build_row_matches_columns(self): + """_build_row returns a list matching SHEET_COLUMNS length.""" + import openjarvis.evals.trackers.sheets_tracker as st_mod + + mock_gspread = MagicMock() + original = st_mod.gspread + st_mod.gspread = mock_gspread + original_creds = st_mod.Credentials + st_mod.Credentials = MagicMock() + try: + tracker = st_mod.SheetsTracker(spreadsheet_id="abc123") + summary = _make_summary() + row = tracker._build_row(summary) + assert len(row) == len(st_mod.SHEET_COLUMNS), ( + f"Row length {len(row)} != columns length {len(st_mod.SHEET_COLUMNS)}" + ) + finally: + st_mod.gspread = original + st_mod.Credentials = original_creds diff --git a/uv.lock b/uv.lock index 6fc7b3df..230605aa 100644 --- a/uv.lock +++ b/uv.lock @@ -4,17 +4,22 @@ requires-python = ">=3.10" resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", "python_full_version >= '3.14' and sys_platform == 'emscripten'", - "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version >= '3.14' and sys_platform == 'linux'", + "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'", "python_full_version == '3.13.*' and sys_platform == 'win32'", "python_full_version == '3.12.*' and sys_platform == 'win32'", "python_full_version == '3.13.*' and sys_platform == 'emscripten'", "python_full_version == '3.12.*' and sys_platform == 'emscripten'", - "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", - "python_full_version == '3.12.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.13.*' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'", + "python_full_version == '3.12.*' and sys_platform == 'linux'", + "python_full_version == '3.12.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'", "python_full_version == '3.11.*' and sys_platform == 'win32'", "python_full_version == '3.11.*' and sys_platform == 'emscripten'", - "python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", - "python_full_version < '3.11'", + "python_full_version == '3.11.*' and sys_platform == 'linux'", + "python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'", + "python_full_version < '3.11' and sys_platform == 'linux'", + "python_full_version < '3.11' and sys_platform != 'linux'", ] [[package]] @@ -1078,7 +1083,7 @@ name = "cuda-bindings" version = "12.9.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cuda-pathfinder", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, + { name = "cuda-pathfinder", marker = "sys_platform == 'linux'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/7a/d8/b546104b8da3f562c1ff8ab36d130c8fe1dd6a045ced80b4f6ad74f7d4e1/cuda_bindings-12.9.4-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4d3c842c2a4303b2a580fe955018e31aea30278be19795ae05226235268032e5", size = 12148218, upload-time = "2025-10-21T14:51:28.855Z" }, @@ -1658,6 +1663,19 @@ requests = [ { name = "requests" }, ] +[[package]] +name = "google-auth-oauthlib" +version = "1.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-auth" }, + { name = "requests-oauthlib" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ac/b4/1b19567e4c567b796f5c593d89895f3cfae5a38e04f27c6af87618fd0942/google_auth_oauthlib-1.3.0.tar.gz", hash = "sha256:cd39e807ac7229d6b8b9c1e297321d36fcc8a9e4857dff4301870985df51a528", size = 21777, upload-time = "2026-02-27T14:13:01.489Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2f/56/909fd5632226d3fba31d7aeffd4754410735d49362f5809956fe3e9af344/google_auth_oauthlib-1.3.0-py3-none-any.whl", hash = "sha256:386b3fb85cf4a5b819c6ad23e3128d975216b4cac76324de1d90b128aaf38f29", size = 19308, upload-time = "2026-02-27T14:12:47.865Z" }, +] + [[package]] name = "google-genai" version = "1.64.0" @@ -1821,6 +1839,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1c/ce/adfe7e5f701d503be7778291757452e3fab6b19acf51917c79f5d1cf7f8a/grpcio-1.78.1-cp314-cp314-win_amd64.whl", hash = "sha256:e2a6b33d1050dce2c6f563c5caf7f7cbeebf7fba8cde37ffe3803d50526900d1", size = 4932000, upload-time = "2026-02-20T01:15:36.127Z" }, ] +[[package]] +name = "gspread" +version = "6.2.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-auth" }, + { name = "google-auth-oauthlib" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/91/83/42d1d813822ed016d77aabadc99b09de3b5bd68532fd6bae23fd62347c41/gspread-6.2.1.tar.gz", hash = "sha256:2c7c99f7c32ebea6ec0d36f2d5cbe8a2be5e8f2a48bde87ad1ea203eff32bd03", size = 82590, upload-time = "2025-05-14T15:56:25.254Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/27/76/563fb20dedd0e12794d9a12cfe0198458cc0501fdc7b034eee2166d035d5/gspread-6.2.1-py3-none-any.whl", hash = "sha256:6d4ec9f1c23ae3c704a9219026dac01f2b328ac70b96f1495055d453c4c184db", size = 59977, upload-time = "2025-05-14T15:56:24.014Z" }, +] + [[package]] name = "h11" version = "0.16.0" @@ -2814,7 +2845,8 @@ name = "networkx" version = "3.4.2" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version < '3.11'", + "python_full_version < '3.11' and sys_platform == 'linux'", + "python_full_version < '3.11' and sys_platform != 'linux'", ] sdist = { url = "https://files.pythonhosted.org/packages/fd/1d/06475e1cd5264c0b870ea2cc6fdb3e37177c1e565c43f56ff17a10e3937f/networkx-3.4.2.tar.gz", hash = "sha256:307c3669428c5362aab27c8a1260aa8f47c4e91d3891f48be0141738d8d053e1", size = 2151368, upload-time = "2024-10-21T12:39:38.695Z" } wheels = [ @@ -2828,16 +2860,20 @@ source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", "python_full_version >= '3.14' and sys_platform == 'emscripten'", - "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version >= '3.14' and sys_platform == 'linux'", + "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'", "python_full_version == '3.13.*' and sys_platform == 'win32'", "python_full_version == '3.12.*' and sys_platform == 'win32'", "python_full_version == '3.13.*' and sys_platform == 'emscripten'", "python_full_version == '3.12.*' and sys_platform == 'emscripten'", - "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", - "python_full_version == '3.12.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.13.*' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'", + "python_full_version == '3.12.*' and sys_platform == 'linux'", + "python_full_version == '3.12.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'", "python_full_version == '3.11.*' and sys_platform == 'win32'", "python_full_version == '3.11.*' and sys_platform == 'emscripten'", - "python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.11.*' and sys_platform == 'linux'", + "python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'", ] sdist = { url = "https://files.pythonhosted.org/packages/6a/51/63fe664f3908c97be9d2e4f1158eb633317598cfa6e1fc14af5383f17512/networkx-3.6.1.tar.gz", hash = "sha256:26b7c357accc0c8cde558ad486283728b65b6a95d85ee1cd66bafab4c8168509", size = 2517025, upload-time = "2025-12-08T17:02:39.908Z" } wheels = [ @@ -2875,7 +2911,8 @@ name = "numpy" version = "2.2.6" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version < '3.11'", + "python_full_version < '3.11' and sys_platform == 'linux'", + "python_full_version < '3.11' and sys_platform != 'linux'", ] sdist = { url = "https://files.pythonhosted.org/packages/76/21/7d2a95e4bba9dc13d043ee156a356c0a8f0c6309dff6b21b4d71a073b8a8/numpy-2.2.6.tar.gz", hash = "sha256:e29554e2bef54a90aa5cc07da6ce955accb83f21ab5de01a62c8478897b264fd", size = 20276440, upload-time = "2025-05-17T22:38:04.611Z" } wheels = [ @@ -2942,16 +2979,20 @@ source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", "python_full_version >= '3.14' and sys_platform == 'emscripten'", - "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version >= '3.14' and sys_platform == 'linux'", + "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'", "python_full_version == '3.13.*' and sys_platform == 'win32'", "python_full_version == '3.12.*' and sys_platform == 'win32'", "python_full_version == '3.13.*' and sys_platform == 'emscripten'", "python_full_version == '3.12.*' and sys_platform == 'emscripten'", - "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", - "python_full_version == '3.12.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.13.*' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'", + "python_full_version == '3.12.*' and sys_platform == 'linux'", + "python_full_version == '3.12.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'", "python_full_version == '3.11.*' and sys_platform == 'win32'", "python_full_version == '3.11.*' and sys_platform == 'emscripten'", - "python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.11.*' and sys_platform == 'linux'", + "python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'", ] sdist = { url = "https://files.pythonhosted.org/packages/57/fd/0005efbd0af48e55eb3c7208af93f2862d4b1a56cd78e84309a2d959208d/numpy-2.4.2.tar.gz", hash = "sha256:659a6107e31a83c4e33f763942275fd278b21d095094044eb35569e86a21ddae", size = 20723651, upload-time = "2026-01-31T23:13:10.135Z" } wheels = [ @@ -3065,7 +3106,7 @@ name = "nvidia-cudnn-cu12" version = "9.10.2.21" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas-cu12", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, + { name = "nvidia-cublas-cu12", marker = "sys_platform == 'linux'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/ba/51/e123d997aa098c61d029f76663dedbfb9bc8dcf8c60cbd6adbe42f76d049/nvidia_cudnn_cu12-9.10.2.21-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:949452be657fa16687d0930933f032835951ef0892b37d2d53824d1a84dc97a8", size = 706758467, upload-time = "2025-06-06T21:54:08.597Z" }, @@ -3076,7 +3117,7 @@ name = "nvidia-cufft-cu12" version = "11.3.3.83" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink-cu12", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, + { name = "nvidia-nvjitlink-cu12", marker = "sys_platform == 'linux'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/1f/13/ee4e00f30e676b66ae65b4f08cb5bcbb8392c03f54f2d5413ea99a5d1c80/nvidia_cufft_cu12-11.3.3.83-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4d2dd21ec0b88cf61b62e6b43564355e5222e4a3fb394cac0db101f2dd0d4f74", size = 193118695, upload-time = "2025-03-07T01:45:27.821Z" }, @@ -3103,9 +3144,9 @@ name = "nvidia-cusolver-cu12" version = "11.7.3.90" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas-cu12", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, - { name = "nvidia-cusparse-cu12", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, - { name = "nvidia-nvjitlink-cu12", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, + { name = "nvidia-cublas-cu12", marker = "sys_platform == 'linux'" }, + { name = "nvidia-cusparse-cu12", marker = "sys_platform == 'linux'" }, + { name = "nvidia-nvjitlink-cu12", marker = "sys_platform == 'linux'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/85/48/9a13d2975803e8cf2777d5ed57b87a0b6ca2cc795f9a4f59796a910bfb80/nvidia_cusolver_cu12-11.7.3.90-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:4376c11ad263152bd50ea295c05370360776f8c3427b30991df774f9fb26c450", size = 267506905, upload-time = "2025-03-07T01:47:16.273Z" }, @@ -3116,7 +3157,7 @@ name = "nvidia-cusparse-cu12" version = "12.5.8.93" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink-cu12", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, + { name = "nvidia-nvjitlink-cu12", marker = "sys_platform == 'linux'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/c2/f5/e1854cb2f2bcd4280c44736c93550cc300ff4b8c95ebe370d0aa7d2b473d/nvidia_cusparse_cu12-12.5.8.93-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1ec05d76bbbd8b61b06a80e1eaf8cf4959c3d4ce8e711b65ebd0443bb0ebb13b", size = 288216466, upload-time = "2025-03-07T01:48:13.779Z" }, @@ -3171,6 +3212,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a2/eb/86626c1bbc2edb86323022371c39aa48df6fd8b0a1647bc274577f72e90b/nvidia_nvtx_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5b17e2001cc0d751a5bc2c6ec6d26ad95913324a4adb86788c944f8ce9ba441f", size = 89954, upload-time = "2025-03-07T01:42:44.131Z" }, ] +[[package]] +name = "oauthlib" +version = "3.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0b/5f/19930f824ffeb0ad4372da4812c50edbd1434f678c90c2733e1188edfc63/oauthlib-3.3.1.tar.gz", hash = "sha256:0f0f8aa759826a193cf66c12ea1af1637f87b9b4622d46e866952bb022e538c9", size = 185918, upload-time = "2025-06-19T22:48:08.269Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/be/9c/92789c596b8df838baa98fa71844d84283302f7604ed565dafe5a6b5041a/oauthlib-3.3.1-py3-none-any.whl", hash = "sha256:88119c938d2b8fb88561af5f6ee0eec8cc8d552b7bb1f712743136eb7523b7a1", size = 160065, upload-time = "2025-06-19T22:48:06.508Z" }, +] + [[package]] name = "onnxruntime" version = "1.24.2" @@ -3346,6 +3396,13 @@ energy-amd = [ energy-apple = [ { name = "zeus-ml" }, ] +eval-sheets = [ + { name = "google-auth" }, + { name = "gspread" }, +] +eval-wandb = [ + { name = "wandb" }, +] gpu-metrics = [ { name = "pynvml" }, ] @@ -3429,7 +3486,9 @@ requires-dist = [ { name = "faiss-cpu", marker = "extra == 'memory-faiss'", specifier = ">=1.7" }, { name = "fastapi", marker = "extra == 'server'", specifier = ">=0.110" }, { name = "faster-whisper", marker = "extra == 'speech'", specifier = ">=1.0" }, + { name = "google-auth", marker = "extra == 'eval-sheets'", specifier = ">=2.0" }, { name = "google-genai", marker = "extra == 'inference-google'", specifier = ">=1.0" }, + { name = "gspread", marker = "extra == 'eval-sheets'", specifier = ">=6.0" }, { name = "httpx", specifier = ">=0.27" }, { name = "line-bot-sdk", marker = "extra == 'channel-line'", specifier = ">=3.0" }, { name = "litellm", marker = "extra == 'inference-litellm'", specifier = ">=1.40" }, @@ -3473,12 +3532,13 @@ requires-dist = [ { name = "twitchio", marker = "extra == 'channel-twitch'", specifier = ">=2.6" }, { name = "uvicorn", marker = "extra == 'server'", specifier = ">=0.30" }, { name = "viberbot", marker = "extra == 'channel-viber'", specifier = ">=1.0" }, + { name = "wandb", marker = "extra == 'eval-wandb'", specifier = ">=0.17" }, { name = "wasmtime", marker = "extra == 'sandbox-wasm'", specifier = ">=25" }, { name = "zeus-ml", extras = ["apple"], marker = "extra == 'energy-all'" }, { name = "zeus-ml", extras = ["apple"], marker = "extra == 'energy-apple'" }, { name = "zulip", marker = "extra == 'channel-zulip'", specifier = ">=0.9" }, ] -provides-extras = ["dev", "inference-ollama", "inference-vllm", "inference-llamacpp", "inference-mlx", "inference-cloud", "inference-google", "inference-litellm", "tools-search", "memory-faiss", "memory-colbert", "memory-pdf", "memory-bm25", "server", "agents", "openhands", "claude-code", "gpu-metrics", "energy-amd", "energy-apple", "energy-all", "learning", "orchestrator-training", "channel-telegram", "channel-discord", "channel-slack", "channel-webhook", "channel-email", "channel-whatsapp", "channel-signal", "channel-google-chat", "channel-irc", "channel-webchat", "channel-teams", "channel-matrix", "channel-mattermost", "channel-feishu", "channel-bluebubbles", "channel-whatsapp-baileys", "channel-line", "channel-viber", "channel-messenger", "channel-reddit", "channel-mastodon", "channel-xmpp", "channel-rocketchat", "channel-zulip", "channel-twitch", "channel-nostr", "browser", "media", "pdf", "scheduler", "security-signing", "sandbox-wasm", "dashboard", "speech", "speech-deepgram", "docs"] +provides-extras = ["dev", "inference-mlx", "inference-cloud", "inference-google", "inference-litellm", "tools-search", "memory-faiss", "memory-colbert", "memory-pdf", "memory-bm25", "server", "openhands", "gpu-metrics", "energy-amd", "energy-apple", "energy-all", "orchestrator-training", "channel-telegram", "channel-discord", "channel-slack", "channel-line", "channel-viber", "channel-messenger", "channel-reddit", "channel-mastodon", "channel-xmpp", "channel-rocketchat", "channel-zulip", "channel-twitch", "channel-nostr", "browser", "media", "pdf", "scheduler", "security-signing", "sandbox-wasm", "dashboard", "speech", "speech-deepgram", "eval-wandb", "eval-sheets", "docs"] [[package]] name = "opentelemetry-api" @@ -3722,7 +3782,8 @@ name = "pandas" version = "2.3.3" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version < '3.11'", + "python_full_version < '3.11' and sys_platform == 'linux'", + "python_full_version < '3.11' and sys_platform != 'linux'", ] dependencies = [ { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, @@ -3788,16 +3849,20 @@ source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", "python_full_version >= '3.14' and sys_platform == 'emscripten'", - "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version >= '3.14' and sys_platform == 'linux'", + "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'", "python_full_version == '3.13.*' and sys_platform == 'win32'", "python_full_version == '3.12.*' and sys_platform == 'win32'", "python_full_version == '3.13.*' and sys_platform == 'emscripten'", "python_full_version == '3.12.*' and sys_platform == 'emscripten'", - "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", - "python_full_version == '3.12.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.13.*' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'", + "python_full_version == '3.12.*' and sys_platform == 'linux'", + "python_full_version == '3.12.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'", "python_full_version == '3.11.*' and sys_platform == 'win32'", "python_full_version == '3.11.*' and sys_platform == 'emscripten'", - "python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.11.*' and sys_platform == 'linux'", + "python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'", ] dependencies = [ { name = "numpy", version = "2.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, @@ -5071,6 +5136,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1e/db/4254e3eabe8020b458f1a747140d32277ec7a271daf1d235b70dc0b4e6e3/requests-2.32.5-py3-none-any.whl", hash = "sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6", size = 64738, upload-time = "2025-08-18T20:46:00.542Z" }, ] +[[package]] +name = "requests-oauthlib" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "oauthlib" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/42/f2/05f29bc3913aea15eb670be136045bf5c5bbf4b99ecb839da9b422bb2c85/requests-oauthlib-2.0.0.tar.gz", hash = "sha256:b3dffaebd884d8cd778494369603a9e7b58d29111bf6b41bdc2dcd87203af4e9", size = 55650, upload-time = "2024-03-22T20:32:29.939Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3b/5d/63d4ae3b9daea098d5d6f5da83984853c1bbacd5dc826764b249fe119d24/requests_oauthlib-2.0.0-py2.py3-none-any.whl", hash = "sha256:7dd8a5c40426b779b0868c404bdef9768deccf22749cde15852df527e6269b36", size = 24179, upload-time = "2024-03-22T20:32:28.055Z" }, +] + [[package]] name = "requests-toolbelt" version = "1.0.0" @@ -5324,7 +5402,8 @@ name = "scikit-learn" version = "1.7.2" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version < '3.11'", + "python_full_version < '3.11' and sys_platform == 'linux'", + "python_full_version < '3.11' and sys_platform != 'linux'", ] dependencies = [ { name = "joblib", marker = "python_full_version < '3.11'" }, @@ -5373,16 +5452,20 @@ source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", "python_full_version >= '3.14' and sys_platform == 'emscripten'", - "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version >= '3.14' and sys_platform == 'linux'", + "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'", "python_full_version == '3.13.*' and sys_platform == 'win32'", "python_full_version == '3.12.*' and sys_platform == 'win32'", "python_full_version == '3.13.*' and sys_platform == 'emscripten'", "python_full_version == '3.12.*' and sys_platform == 'emscripten'", - "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", - "python_full_version == '3.12.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.13.*' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'", + "python_full_version == '3.12.*' and sys_platform == 'linux'", + "python_full_version == '3.12.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'", "python_full_version == '3.11.*' and sys_platform == 'win32'", "python_full_version == '3.11.*' and sys_platform == 'emscripten'", - "python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.11.*' and sys_platform == 'linux'", + "python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'", ] dependencies = [ { name = "joblib", marker = "python_full_version >= '3.11'" }, @@ -5435,7 +5518,8 @@ name = "scipy" version = "1.15.3" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version < '3.11'", + "python_full_version < '3.11' and sys_platform == 'linux'", + "python_full_version < '3.11' and sys_platform != 'linux'", ] dependencies = [ { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, @@ -5496,16 +5580,20 @@ source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", "python_full_version >= '3.14' and sys_platform == 'emscripten'", - "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version >= '3.14' and sys_platform == 'linux'", + "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'", "python_full_version == '3.13.*' and sys_platform == 'win32'", "python_full_version == '3.12.*' and sys_platform == 'win32'", "python_full_version == '3.13.*' and sys_platform == 'emscripten'", "python_full_version == '3.12.*' and sys_platform == 'emscripten'", - "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", - "python_full_version == '3.12.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.13.*' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'", + "python_full_version == '3.12.*' and sys_platform == 'linux'", + "python_full_version == '3.12.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'", "python_full_version == '3.11.*' and sys_platform == 'win32'", "python_full_version == '3.11.*' and sys_platform == 'emscripten'", - "python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.11.*' and sys_platform == 'linux'", + "python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'", ] dependencies = [ { name = "numpy", version = "2.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, @@ -5579,8 +5667,8 @@ name = "secretstorage" version = "3.5.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cryptography", marker = "python_full_version >= '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, - { name = "jeepney", marker = "python_full_version >= '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "cryptography", marker = "python_full_version >= '3.12' and sys_platform == 'linux'" }, + { name = "jeepney", marker = "python_full_version >= '3.12' and sys_platform == 'linux'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/1c/03/e834bcd866f2f8a49a85eaff47340affa3bfa391ee9912a952a1faa68c7b/secretstorage-3.5.0.tar.gz", hash = "sha256:f04b8e4689cbce351744d5537bf6b1329c6fc68f91fa666f60a380edddcd11be", size = 19884, upload-time = "2025-11-23T19:02:53.191Z" } wheels = [ @@ -5638,6 +5726,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/03/b0/811dae8fb9f2784e138785d481469788f2e0d0c109c5737372454415f55f/sentencepiece-0.2.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:477c81505db072b3ab627e7eab972ea1025331bd3a92bacbf798df2b75ea86ec", size = 1254846, upload-time = "2025-08-12T07:00:40.611Z" }, ] +[[package]] +name = "sentry-sdk" +version = "2.54.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c8/e9/2e3a46c304e7fa21eaa70612f60354e32699c7102eb961f67448e222ad7c/sentry_sdk-2.54.0.tar.gz", hash = "sha256:2620c2575128d009b11b20f7feb81e4e4e8ae08ec1d36cbc845705060b45cc1b", size = 413813, upload-time = "2026-03-02T15:12:41.355Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/53/39/be412cc86bc6247b8f69e9383d7950711bd86f8d0a4a4b0fe8fad685bc21/sentry_sdk-2.54.0-py2.py3-none-any.whl", hash = "sha256:fd74e0e281dcda63afff095d23ebcd6e97006102cdc8e78a29f19ecdf796a0de", size = 439198, upload-time = "2026-03-02T15:12:39.546Z" }, +] + [[package]] name = "setuptools" version = "82.0.0" @@ -5679,7 +5780,8 @@ name = "slixmpp" version = "1.12.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version < '3.11'", + "python_full_version < '3.11' and sys_platform == 'linux'", + "python_full_version < '3.11' and sys_platform != 'linux'", ] dependencies = [ { name = "aiodns", marker = "python_full_version < '3.11'" }, @@ -5717,16 +5819,20 @@ source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", "python_full_version >= '3.14' and sys_platform == 'emscripten'", - "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version >= '3.14' and sys_platform == 'linux'", + "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'", "python_full_version == '3.13.*' and sys_platform == 'win32'", "python_full_version == '3.12.*' and sys_platform == 'win32'", "python_full_version == '3.13.*' and sys_platform == 'emscripten'", "python_full_version == '3.12.*' and sys_platform == 'emscripten'", - "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", - "python_full_version == '3.12.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.13.*' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'", + "python_full_version == '3.12.*' and sys_platform == 'linux'", + "python_full_version == '3.12.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'", "python_full_version == '3.11.*' and sys_platform == 'win32'", "python_full_version == '3.11.*' and sys_platform == 'emscripten'", - "python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.11.*' and sys_platform == 'linux'", + "python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'", ] dependencies = [ { name = "aiodns", marker = "python_full_version >= '3.11'" }, @@ -6146,7 +6252,8 @@ name = "twitchio" version = "2.10.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version < '3.11'", + "python_full_version < '3.11' and sys_platform == 'linux'", + "python_full_version < '3.11' and sys_platform != 'linux'", ] dependencies = [ { name = "aiohttp", marker = "python_full_version < '3.11'" }, @@ -6165,16 +6272,20 @@ source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", "python_full_version >= '3.14' and sys_platform == 'emscripten'", - "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version >= '3.14' and sys_platform == 'linux'", + "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'", "python_full_version == '3.13.*' and sys_platform == 'win32'", "python_full_version == '3.12.*' and sys_platform == 'win32'", "python_full_version == '3.13.*' and sys_platform == 'emscripten'", "python_full_version == '3.12.*' and sys_platform == 'emscripten'", - "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", - "python_full_version == '3.12.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.13.*' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'", + "python_full_version == '3.12.*' and sys_platform == 'linux'", + "python_full_version == '3.12.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'", "python_full_version == '3.11.*' and sys_platform == 'win32'", "python_full_version == '3.11.*' and sys_platform == 'emscripten'", - "python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.11.*' and sys_platform == 'linux'", + "python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'", ] dependencies = [ { name = "aiohttp", marker = "python_full_version >= '3.11'" }, @@ -6404,6 +6515,35 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5d/c1/f01846f10c8383b197a67b4cdfea6eea476aab678ed9f6f5d9ccc8c8dddf/viberbot-1.0.12-py3-none-any.whl", hash = "sha256:ca43fea2945d650c2ef2cbd777f3c546c795bf945278f6620ceda42f4101d801", size = 26164, upload-time = "2021-01-27T13:45:43.726Z" }, ] +[[package]] +name = "wandb" +version = "0.25.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "gitpython" }, + { name = "packaging" }, + { name = "platformdirs" }, + { name = "protobuf" }, + { name = "pydantic" }, + { name = "pyyaml" }, + { name = "requests" }, + { name = "sentry-sdk" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fd/60/d94952549920469524b689479c864c692ca47eca4b8c2fe3389b64a58778/wandb-0.25.0.tar.gz", hash = "sha256:45840495a288e34245d69d07b5a0b449220fbc5b032e6b51c4f92ec9026d2ad1", size = 43951335, upload-time = "2026-02-13T00:17:45.515Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/7d/0c131db3ec9deaabbd32263d90863cbfbe07659527e11c35a5c738cecdc5/wandb-0.25.0-py3-none-macosx_12_0_arm64.whl", hash = "sha256:5eecb3c7b5e60d1acfa4b056bfbaa0b79a482566a9db58c9f99724b3862bc8e5", size = 23287536, upload-time = "2026-02-13T00:17:20.265Z" }, + { url = "https://files.pythonhosted.org/packages/c3/95/31bb7f76a966ec87495e5a72ac7570685be162494c41757ac871768dbc4f/wandb-0.25.0-py3-none-macosx_12_0_x86_64.whl", hash = "sha256:daeedaadb183dc466e634fba90ab2bab1d4e93000912be0dee95065a0624a3fd", size = 25196062, upload-time = "2026-02-13T00:17:23.356Z" }, + { url = "https://files.pythonhosted.org/packages/d9/a1/258cdedbf30cebc692198a774cf0ef945b7ed98ee64bdaf62621281c95d8/wandb-0.25.0-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:5e0127dbcef13eea48f4b84268da7004d34d3120ebc7b2fa9cefb72b49dbb825", size = 22799744, upload-time = "2026-02-13T00:17:26.437Z" }, + { url = "https://files.pythonhosted.org/packages/de/91/ec9465d014cfd199c5b2083d271d31b3c2aedeae66f3d8a0712f7f54bdf3/wandb-0.25.0-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:6c4c38077836f9b7569a35b0e1dcf1f0c43616fcd936d182f475edbfea063665", size = 25262839, upload-time = "2026-02-13T00:17:28.8Z" }, + { url = "https://files.pythonhosted.org/packages/c7/95/cb2d1c7143f534544147fb53fe87944508b8cb9a058bc5b6f8a94adbee15/wandb-0.25.0-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:6edd8948d305cb73745bf564b807bd73da2ccbd47c548196b8a362f7df40aed8", size = 22853714, upload-time = "2026-02-13T00:17:31.68Z" }, + { url = "https://files.pythonhosted.org/packages/d7/94/68163f70c1669edcf130822aaaea782d8198b5df74443eca0085ec596774/wandb-0.25.0-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:ada6f08629bb014ad6e0a19d5dec478cdaa116431baa3f0a4bf4ab8d9893611f", size = 25358037, upload-time = "2026-02-13T00:17:34.676Z" }, + { url = "https://files.pythonhosted.org/packages/cc/fb/9578eed2c01b2fc6c8b693da110aa9c73a33d7bb556480f5cfc42e48c94e/wandb-0.25.0-py3-none-win32.whl", hash = "sha256:020b42ca4d76e347709d65f59b30d4623a115edc28f462af1c92681cb17eae7c", size = 24604118, upload-time = "2026-02-13T00:17:37.641Z" }, + { url = "https://files.pythonhosted.org/packages/25/97/460f6cb738aaa39b4eb2e6b4c630b2ae4321cdd70a79d5955ea75a878981/wandb-0.25.0-py3-none-win_amd64.whl", hash = "sha256:78307ac0b328f2dc334c8607bec772851215584b62c439eb320c4af4fb077a00", size = 24604122, upload-time = "2026-02-13T00:17:39.991Z" }, + { url = "https://files.pythonhosted.org/packages/27/6c/5847b4dda1dfd52630dac08711d4348c69ed657f0698fc2d949c7f7a6622/wandb-0.25.0-py3-none-win_arm64.whl", hash = "sha256:c6174401fd6fb726295e98d57b4231c100eca96bd17de51bfc64038a57230aaf", size = 21785298, upload-time = "2026-02-13T00:17:42.475Z" }, +] + [[package]] name = "wasmtime" version = "42.0.0"