mirror of
https://github.com/open-jarvis/OpenJarvis.git
synced 2026-07-30 10:52:15 +00:00
Merge pull request #6 from HazyResearch/feat/eval-trackers
feat(evals): add W&B and Google Sheets result trackers
This commit is contained in:
Jon Saad-Falcon
@@ -80,6 +80,8 @@ sandbox-wasm = ["wasmtime>=25"]
|
||||
dashboard = ["textual>=0.80"]
|
||||
speech = ["faster-whisper>=1.0"]
|
||||
speech-deepgram = ["deepgram-sdk>=3.0"]
|
||||
eval-wandb = ["wandb>=0.17"]
|
||||
eval-sheets = ["gspread>=6.0", "google-auth>=2.0"]
|
||||
docs = [
|
||||
"mkdocs>=1.6",
|
||||
"mkdocs-material>=9.5",
|
||||
|
||||
@@ -113,6 +113,34 @@ def eval_list() -> None:
|
||||
"-o", "--output", "output_path", default=None, type=click.Path(),
|
||||
help="Output JSONL path.",
|
||||
)
|
||||
@click.option(
|
||||
"--wandb-project", "wandb_project", default="",
|
||||
help="W&B project name (enables W&B tracking).",
|
||||
)
|
||||
@click.option(
|
||||
"--wandb-entity", "wandb_entity", default="",
|
||||
help="W&B entity (team or user).",
|
||||
)
|
||||
@click.option(
|
||||
"--wandb-tags", "wandb_tags", default="",
|
||||
help="Comma-separated W&B tags.",
|
||||
)
|
||||
@click.option(
|
||||
"--wandb-group", "wandb_group", default="",
|
||||
help="W&B run group.",
|
||||
)
|
||||
@click.option(
|
||||
"--sheets-id", "sheets_spreadsheet_id", default="",
|
||||
help="Google Sheets spreadsheet ID.",
|
||||
)
|
||||
@click.option(
|
||||
"--sheets-worksheet", "sheets_worksheet", default="Results",
|
||||
help="Google Sheets worksheet name.",
|
||||
)
|
||||
@click.option(
|
||||
"--sheets-creds", "sheets_credentials_path", default="",
|
||||
help="Path to Google service account JSON.",
|
||||
)
|
||||
@click.option(
|
||||
"-v", "--verbose", "verbose", is_flag=True, default=False,
|
||||
help="Verbose logging.",
|
||||
@@ -127,6 +155,13 @@ def eval_run(
|
||||
tools: str,
|
||||
telemetry: bool,
|
||||
output_path: Optional[str],
|
||||
wandb_project: str,
|
||||
wandb_entity: str,
|
||||
wandb_tags: str,
|
||||
wandb_group: str,
|
||||
sheets_spreadsheet_id: str,
|
||||
sheets_worksheet: str,
|
||||
sheets_credentials_path: str,
|
||||
verbose: bool,
|
||||
) -> None:
|
||||
"""Run evaluation benchmarks."""
|
||||
@@ -216,6 +251,13 @@ def eval_run(
|
||||
tools=tool_list,
|
||||
output_path=output_path,
|
||||
telemetry=telemetry,
|
||||
wandb_project=wandb_project,
|
||||
wandb_entity=wandb_entity,
|
||||
wandb_tags=wandb_tags,
|
||||
wandb_group=wandb_group,
|
||||
sheets_spreadsheet_id=sheets_spreadsheet_id,
|
||||
sheets_worksheet=sheets_worksheet,
|
||||
sheets_credentials_path=sheets_credentials_path,
|
||||
)
|
||||
|
||||
try:
|
||||
|
||||
@@ -217,6 +217,39 @@ def _print_summary(
|
||||
print_completion(console, summary, output_path, traces_dir)
|
||||
|
||||
|
||||
def _build_trackers(config) -> list:
|
||||
"""Build tracker instances from RunConfig fields."""
|
||||
trackers = []
|
||||
if getattr(config, "wandb_project", ""):
|
||||
try:
|
||||
from openjarvis.evals.trackers.wandb_tracker import WandbTracker
|
||||
trackers.append(WandbTracker(
|
||||
project=config.wandb_project,
|
||||
entity=getattr(config, "wandb_entity", ""),
|
||||
tags=getattr(config, "wandb_tags", ""),
|
||||
group=getattr(config, "wandb_group", ""),
|
||||
))
|
||||
except ImportError as exc:
|
||||
raise click.ClickException(
|
||||
f"wandb not installed: {exc}\n"
|
||||
"Install with: pip install 'openjarvis[eval-wandb]'"
|
||||
) from exc
|
||||
if getattr(config, "sheets_spreadsheet_id", ""):
|
||||
try:
|
||||
from openjarvis.evals.trackers.sheets_tracker import SheetsTracker
|
||||
trackers.append(SheetsTracker(
|
||||
spreadsheet_id=config.sheets_spreadsheet_id,
|
||||
worksheet=getattr(config, "sheets_worksheet", "Results"),
|
||||
credentials_path=getattr(config, "sheets_credentials_path", ""),
|
||||
))
|
||||
except ImportError as exc:
|
||||
raise click.ClickException(
|
||||
f"gspread not installed: {exc}\n"
|
||||
"Install with: pip install 'openjarvis[eval-sheets]'"
|
||||
) from exc
|
||||
return trackers
|
||||
|
||||
|
||||
def _run_single(config, console: Optional[Console] = None) -> object:
|
||||
"""Run a single eval from a RunConfig and return the summary."""
|
||||
from openjarvis.evals.core.runner import EvalRunner
|
||||
@@ -236,7 +269,8 @@ def _run_single(config, console: Optional[Console] = None) -> object:
|
||||
judge_backend = _build_judge_backend(config.judge_model)
|
||||
scorer = _build_scorer(config.benchmark, judge_backend, config.judge_model)
|
||||
|
||||
runner = EvalRunner(config, dataset, eval_backend, scorer)
|
||||
trackers = _build_trackers(config)
|
||||
runner = EvalRunner(config, dataset, eval_backend, scorer, trackers=trackers)
|
||||
try:
|
||||
num_samples = config.max_samples or 0
|
||||
# Use progress bar if we know the sample count
|
||||
@@ -292,6 +326,11 @@ def _run_from_config(config_path: str, verbose: bool) -> None:
|
||||
output_dir = Path(suite.run.output_dir)
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Auto-set wandb_group to suite name if W&B enabled and no explicit group
|
||||
for rc in run_configs:
|
||||
if rc.wandb_project and not rc.wandb_group:
|
||||
rc.wandb_group = suite_name
|
||||
|
||||
summaries = []
|
||||
for i, rc in enumerate(run_configs, 1):
|
||||
print_section(
|
||||
@@ -355,12 +394,30 @@ def main():
|
||||
help="Enable GPU metrics collection")
|
||||
@click.option("--compact", is_flag=True, default=False, help="Dense single-table output")
|
||||
@click.option("--trace-detail", is_flag=True, default=False, help="Full per-step trace listing")
|
||||
@click.option("--wandb-project", default="",
|
||||
help="W&B project name (enables tracking)")
|
||||
@click.option("--wandb-entity", default="",
|
||||
help="W&B entity (team or user)")
|
||||
@click.option("--wandb-tags", default="",
|
||||
help="Comma-separated W&B tags")
|
||||
@click.option("--wandb-group", default="",
|
||||
help="W&B run group")
|
||||
@click.option("--sheets-id", "sheets_spreadsheet_id", default="",
|
||||
help="Google Sheets spreadsheet ID")
|
||||
@click.option("--sheets-worksheet", default="Results",
|
||||
help="Google Sheets worksheet name")
|
||||
@click.option("--sheets-creds", "sheets_credentials_path",
|
||||
default="",
|
||||
help="Service account JSON path")
|
||||
@click.option("-v", "--verbose", is_flag=True, help="Verbose logging")
|
||||
@click.pass_context
|
||||
def run(ctx, config_path, benchmark, backend, model, engine_key, agent_name,
|
||||
tools, max_samples, max_workers, judge_model, output_path, seed,
|
||||
dataset_split, temperature, max_tokens, telemetry, gpu_metrics,
|
||||
compact, trace_detail, verbose):
|
||||
compact, trace_detail,
|
||||
wandb_project, wandb_entity, wandb_tags, wandb_group,
|
||||
sheets_spreadsheet_id, sheets_worksheet, sheets_credentials_path,
|
||||
verbose):
|
||||
"""Run a single benchmark evaluation, or a full suite from a TOML config."""
|
||||
_setup_logging(verbose)
|
||||
|
||||
@@ -404,6 +461,13 @@ def run(ctx, config_path, benchmark, backend, model, engine_key, agent_name,
|
||||
dataset_split=dataset_split,
|
||||
telemetry=telemetry,
|
||||
gpu_metrics=gpu_metrics,
|
||||
wandb_project=wandb_project,
|
||||
wandb_entity=wandb_entity,
|
||||
wandb_tags=wandb_tags,
|
||||
wandb_group=wandb_group,
|
||||
sheets_spreadsheet_id=sheets_spreadsheet_id,
|
||||
sheets_worksheet=sheets_worksheet,
|
||||
sheets_credentials_path=sheets_credentials_path,
|
||||
)
|
||||
|
||||
# Banner + config
|
||||
@@ -493,7 +557,8 @@ def run_all(model, engine_key, max_samples, max_workers, judge_model,
|
||||
judge_backend = _build_judge_backend(judge_model)
|
||||
scorer = _build_scorer(bench_name, judge_backend, judge_model)
|
||||
|
||||
runner = EvalRunner(config, dataset, eval_backend, scorer)
|
||||
trackers = _build_trackers(config)
|
||||
runner = EvalRunner(config, dataset, eval_backend, scorer, trackers=trackers)
|
||||
try:
|
||||
if max_samples and max_samples > 0:
|
||||
with Progress(
|
||||
|
||||
@@ -99,6 +99,13 @@ def load_eval_config(path: str | Path) -> EvalSuiteConfig:
|
||||
gpu_metrics=bool(run_raw.get("gpu_metrics", False)),
|
||||
warmup_samples=int(run_raw.get("warmup_samples", 0)),
|
||||
energy_vendor=run_raw.get("energy_vendor", ""),
|
||||
wandb_project=run_raw.get("wandb_project", ""),
|
||||
wandb_entity=run_raw.get("wandb_entity", ""),
|
||||
wandb_tags=run_raw.get("wandb_tags", ""),
|
||||
wandb_group=run_raw.get("wandb_group", ""),
|
||||
sheets_spreadsheet_id=run_raw.get("sheets_spreadsheet_id", ""),
|
||||
sheets_worksheet=run_raw.get("sheets_worksheet", "Results"),
|
||||
sheets_credentials_path=run_raw.get("sheets_credentials_path", ""),
|
||||
)
|
||||
|
||||
# Parse [[models]]
|
||||
@@ -243,6 +250,13 @@ def expand_suite(suite: EvalSuiteConfig) -> List[RunConfig]:
|
||||
gpu_metrics=suite.run.gpu_metrics,
|
||||
metadata=model_meta,
|
||||
warmup_samples=suite.run.warmup_samples,
|
||||
wandb_project=suite.run.wandb_project,
|
||||
wandb_entity=suite.run.wandb_entity,
|
||||
wandb_tags=suite.run.wandb_tags,
|
||||
wandb_group=suite.run.wandb_group,
|
||||
sheets_spreadsheet_id=suite.run.sheets_spreadsheet_id,
|
||||
sheets_worksheet=suite.run.sheets_worksheet,
|
||||
sheets_credentials_path=suite.run.sheets_credentials_path,
|
||||
))
|
||||
|
||||
return configs
|
||||
|
||||
@@ -14,7 +14,14 @@ from typing import Any, Callable, Dict, List, Optional
|
||||
from openjarvis.evals.core.backend import InferenceBackend
|
||||
from openjarvis.evals.core.dataset import DatasetProvider
|
||||
from openjarvis.evals.core.scorer import Scorer
|
||||
from openjarvis.evals.core.types import EvalRecord, EvalResult, MetricStats, RunConfig, RunSummary
|
||||
from openjarvis.evals.core.tracker import ResultTracker
|
||||
from openjarvis.evals.core.types import (
|
||||
EvalRecord,
|
||||
EvalResult,
|
||||
MetricStats,
|
||||
RunConfig,
|
||||
RunSummary,
|
||||
)
|
||||
|
||||
try:
|
||||
from openjarvis.telemetry.efficiency import compute_efficiency
|
||||
@@ -33,11 +40,13 @@ class EvalRunner:
|
||||
dataset: DatasetProvider,
|
||||
backend: InferenceBackend,
|
||||
scorer: Scorer,
|
||||
trackers: Optional[List[ResultTracker]] = None,
|
||||
) -> None:
|
||||
self._config = config
|
||||
self._dataset = dataset
|
||||
self._backend = backend
|
||||
self._scorer = scorer
|
||||
self._trackers: List[ResultTracker] = trackers or []
|
||||
self._results: List[EvalResult] = []
|
||||
self._output_file: Optional[Any] = None
|
||||
|
||||
@@ -79,6 +88,16 @@ class EvalRunner:
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self._output_file = open(output_path, "w")
|
||||
|
||||
# Notify trackers of run start
|
||||
for tracker in self._trackers:
|
||||
try:
|
||||
tracker.on_run_start(cfg)
|
||||
except Exception as exc:
|
||||
LOGGER.warning(
|
||||
"Tracker %s.on_run_start failed: %s",
|
||||
type(tracker).__name__, exc,
|
||||
)
|
||||
|
||||
total = len(records)
|
||||
try:
|
||||
with ThreadPoolExecutor(max_workers=cfg.max_workers) as pool:
|
||||
@@ -99,6 +118,23 @@ class EvalRunner:
|
||||
ended_at = time.time()
|
||||
summary = self._compute_summary(records, started_at, ended_at)
|
||||
|
||||
# Notify trackers of summary and run end
|
||||
for tracker in self._trackers:
|
||||
try:
|
||||
tracker.on_summary(summary)
|
||||
except Exception as exc:
|
||||
LOGGER.warning(
|
||||
"Tracker %s.on_summary failed: %s",
|
||||
type(tracker).__name__, exc,
|
||||
)
|
||||
try:
|
||||
tracker.on_run_end()
|
||||
except Exception as exc:
|
||||
LOGGER.warning(
|
||||
"Tracker %s.on_run_end failed: %s",
|
||||
type(tracker).__name__, exc,
|
||||
)
|
||||
|
||||
# Write summary JSON alongside JSONL
|
||||
traces_dir: Optional[Path] = None
|
||||
if output_path:
|
||||
@@ -255,6 +291,16 @@ class EvalRunner:
|
||||
self._output_file.write(json.dumps(record_dict) + "\n")
|
||||
self._output_file.flush()
|
||||
|
||||
# Notify trackers of each result
|
||||
for tracker in self._trackers:
|
||||
try:
|
||||
tracker.on_result(result, self._config)
|
||||
except Exception as exc:
|
||||
LOGGER.warning(
|
||||
"Tracker %s.on_result failed: %s",
|
||||
type(tracker).__name__, exc,
|
||||
)
|
||||
|
||||
def _resolve_output_path(self) -> Optional[Path]:
|
||||
"""Determine the output file path."""
|
||||
if self._config.output_path:
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
"""ResultTracker ABC for external experiment tracking."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
from openjarvis.evals.core.types import EvalResult, RunConfig, RunSummary
|
||||
|
||||
|
||||
class ResultTracker(ABC):
|
||||
"""Abstract base class for experiment result trackers.
|
||||
|
||||
Lifecycle: on_run_start -> on_result (per sample)
|
||||
-> on_summary -> on_run_end.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def on_run_start(self, config: RunConfig) -> None:
|
||||
"""Called once before evaluation begins."""
|
||||
|
||||
@abstractmethod
|
||||
def on_result(self, result: EvalResult, config: RunConfig) -> None:
|
||||
"""Called after each sample is evaluated."""
|
||||
|
||||
@abstractmethod
|
||||
def on_summary(self, summary: RunSummary) -> None:
|
||||
"""Called after all samples are evaluated with aggregate stats."""
|
||||
|
||||
@abstractmethod
|
||||
def on_run_end(self) -> None:
|
||||
"""Called at the very end of a run for cleanup."""
|
||||
|
||||
|
||||
__all__ = ["ResultTracker"]
|
||||
@@ -71,6 +71,13 @@ class RunConfig:
|
||||
gpu_metrics: bool = False
|
||||
metadata: Dict[str, Any] = field(default_factory=dict)
|
||||
warmup_samples: int = 0
|
||||
wandb_project: str = ""
|
||||
wandb_entity: str = ""
|
||||
wandb_tags: str = ""
|
||||
wandb_group: str = ""
|
||||
sheets_spreadsheet_id: str = ""
|
||||
sheets_worksheet: str = "Results"
|
||||
sheets_credentials_path: str = ""
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
@@ -176,6 +183,13 @@ class ExecutionConfig:
|
||||
gpu_metrics: bool = False
|
||||
warmup_samples: int = 0
|
||||
energy_vendor: str = ""
|
||||
wandb_project: str = ""
|
||||
wandb_entity: str = ""
|
||||
wandb_tags: str = ""
|
||||
wandb_group: str = ""
|
||||
sheets_spreadsheet_id: str = ""
|
||||
sheets_worksheet: str = "Results"
|
||||
sheets_credentials_path: str = ""
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
"""External experiment trackers for the eval framework.
|
||||
|
||||
Trackers are lazily imported to avoid mandatory dependencies on wandb/gspread.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
def WandbTracker(*args, **kwargs): # noqa: N802
|
||||
"""Lazy constructor — imports the real class on first use."""
|
||||
from openjarvis.evals.trackers.wandb_tracker import WandbTracker as _Cls
|
||||
|
||||
return _Cls(*args, **kwargs)
|
||||
|
||||
|
||||
def SheetsTracker(*args, **kwargs): # noqa: N802
|
||||
"""Lazy constructor — imports the real class on first use."""
|
||||
from openjarvis.evals.trackers.sheets_tracker import SheetsTracker as _Cls
|
||||
|
||||
return _Cls(*args, **kwargs)
|
||||
|
||||
|
||||
__all__ = ["WandbTracker", "SheetsTracker"]
|
||||
@@ -0,0 +1,162 @@
|
||||
"""Google Sheets experiment tracker for the eval framework."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
from typing import Any, List, Optional
|
||||
|
||||
from openjarvis.evals.core.tracker import ResultTracker
|
||||
from openjarvis.evals.core.types import EvalResult, MetricStats, RunConfig, RunSummary
|
||||
|
||||
try:
|
||||
import gspread
|
||||
from google.oauth2.service_account import Credentials
|
||||
except ImportError:
|
||||
gspread = None # type: ignore[assignment]
|
||||
Credentials = None # type: ignore[assignment,misc]
|
||||
|
||||
LOGGER = logging.getLogger(__name__)
|
||||
|
||||
# Canonical column order for the summary row.
|
||||
SHEET_COLUMNS: List[str] = [
|
||||
"timestamp",
|
||||
"benchmark",
|
||||
"model",
|
||||
"backend",
|
||||
"total_samples",
|
||||
"scored_samples",
|
||||
"correct",
|
||||
"accuracy",
|
||||
"errors",
|
||||
"mean_latency_seconds",
|
||||
"total_cost_usd",
|
||||
"total_energy_joules",
|
||||
"avg_power_watts",
|
||||
"total_input_tokens",
|
||||
"total_output_tokens",
|
||||
"latency_mean",
|
||||
"latency_p90",
|
||||
"latency_p95",
|
||||
"energy_mean",
|
||||
"energy_p90",
|
||||
"throughput_mean",
|
||||
"throughput_p90",
|
||||
"ipw_mean",
|
||||
"ipj_mean",
|
||||
"mfu_mean",
|
||||
"mbu_mean",
|
||||
"ttft_mean",
|
||||
"ttft_p90",
|
||||
"gpu_utilization_mean",
|
||||
]
|
||||
|
||||
|
||||
def _stat_val(ms: Optional[MetricStats], attr: str) -> Any:
|
||||
"""Safely extract a stat value from a MetricStats, returning '' if None."""
|
||||
if ms is None:
|
||||
return ""
|
||||
return getattr(ms, attr, "")
|
||||
|
||||
|
||||
class SheetsTracker(ResultTracker):
|
||||
"""Appends a summary row to a Google Sheet after each eval run."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
spreadsheet_id: str,
|
||||
worksheet: str = "Results",
|
||||
credentials_path: str = "",
|
||||
) -> None:
|
||||
if gspread is None:
|
||||
raise ImportError(
|
||||
"gspread is not installed. "
|
||||
"Install it with: pip install 'openjarvis[eval-sheets]'"
|
||||
)
|
||||
self._spreadsheet_id = spreadsheet_id
|
||||
self._worksheet_name = worksheet
|
||||
self._credentials_path = credentials_path
|
||||
|
||||
def on_run_start(self, config: RunConfig) -> None:
|
||||
pass
|
||||
|
||||
def on_result(self, result: EvalResult, config: RunConfig) -> None:
|
||||
# No-op: summary-only to avoid excessive API calls.
|
||||
pass
|
||||
|
||||
def on_summary(self, summary: RunSummary) -> None:
|
||||
row = self._build_row(summary)
|
||||
try:
|
||||
gc = self._authorize()
|
||||
spreadsheet = gc.open_by_key(self._spreadsheet_id)
|
||||
try:
|
||||
ws = spreadsheet.worksheet(self._worksheet_name)
|
||||
except gspread.exceptions.WorksheetNotFound:
|
||||
ws = spreadsheet.add_worksheet(
|
||||
title=self._worksheet_name, rows=1000, cols=len(SHEET_COLUMNS),
|
||||
)
|
||||
# Ensure header row exists (idempotent)
|
||||
existing = ws.row_values(1)
|
||||
if not existing or existing[0] != SHEET_COLUMNS[0]:
|
||||
ws.update(range_name="A1", values=[SHEET_COLUMNS])
|
||||
ws.append_row(row, value_input_option="RAW")
|
||||
LOGGER.info("Appended summary row to Google Sheet")
|
||||
except Exception as exc:
|
||||
LOGGER.warning("SheetsTracker.on_summary failed: %s", exc)
|
||||
|
||||
def on_run_end(self) -> None:
|
||||
pass
|
||||
|
||||
def _authorize(self):
|
||||
"""Authenticate with Google Sheets API."""
|
||||
scopes = [
|
||||
"https://www.googleapis.com/auth/spreadsheets",
|
||||
"https://www.googleapis.com/auth/drive",
|
||||
]
|
||||
if self._credentials_path:
|
||||
creds = Credentials.from_service_account_file(
|
||||
self._credentials_path, scopes=scopes,
|
||||
)
|
||||
else:
|
||||
# Fall back to Application Default Credentials
|
||||
import google.auth
|
||||
|
||||
creds, _ = google.auth.default(scopes=scopes)
|
||||
return gspread.authorize(creds)
|
||||
|
||||
def _build_row(self, s: RunSummary) -> List[Any]:
|
||||
"""Build a flat row matching SHEET_COLUMNS order."""
|
||||
return [
|
||||
time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||||
s.benchmark,
|
||||
s.model,
|
||||
s.backend,
|
||||
s.total_samples,
|
||||
s.scored_samples,
|
||||
s.correct,
|
||||
s.accuracy,
|
||||
s.errors,
|
||||
s.mean_latency_seconds,
|
||||
s.total_cost_usd,
|
||||
s.total_energy_joules,
|
||||
s.avg_power_watts,
|
||||
s.total_input_tokens,
|
||||
s.total_output_tokens,
|
||||
_stat_val(s.latency_stats, "mean"),
|
||||
_stat_val(s.latency_stats, "p90"),
|
||||
_stat_val(s.latency_stats, "p95"),
|
||||
_stat_val(s.energy_stats, "mean"),
|
||||
_stat_val(s.energy_stats, "p90"),
|
||||
_stat_val(s.throughput_stats, "mean"),
|
||||
_stat_val(s.throughput_stats, "p90"),
|
||||
_stat_val(s.ipw_stats, "mean"),
|
||||
_stat_val(s.ipj_stats, "mean"),
|
||||
_stat_val(s.mfu_stats, "mean"),
|
||||
_stat_val(s.mbu_stats, "mean"),
|
||||
_stat_val(s.ttft_stats, "mean"),
|
||||
_stat_val(s.ttft_stats, "p90"),
|
||||
_stat_val(s.gpu_utilization_stats, "mean"),
|
||||
]
|
||||
|
||||
|
||||
__all__ = ["SheetsTracker", "SHEET_COLUMNS"]
|
||||
@@ -0,0 +1,154 @@
|
||||
"""W&B experiment tracker for the eval framework."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from openjarvis.evals.core.tracker import ResultTracker
|
||||
from openjarvis.evals.core.types import EvalResult, MetricStats, RunConfig, RunSummary
|
||||
|
||||
try:
|
||||
import wandb
|
||||
except ImportError:
|
||||
wandb = None # type: ignore[assignment]
|
||||
|
||||
LOGGER = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _flatten_metric_stats(prefix: str, ms: Optional[MetricStats]) -> Dict[str, float]:
|
||||
"""Flatten a MetricStats into a dict with prefixed keys."""
|
||||
if ms is None:
|
||||
return {}
|
||||
return {
|
||||
f"{prefix}_mean": ms.mean,
|
||||
f"{prefix}_median": ms.median,
|
||||
f"{prefix}_min": ms.min,
|
||||
f"{prefix}_max": ms.max,
|
||||
f"{prefix}_std": ms.std,
|
||||
f"{prefix}_p90": ms.p90,
|
||||
f"{prefix}_p95": ms.p95,
|
||||
f"{prefix}_p99": ms.p99,
|
||||
}
|
||||
|
||||
|
||||
class WandbTracker(ResultTracker):
|
||||
"""Streams per-sample metrics to Weights & Biases."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
project: str,
|
||||
entity: str = "",
|
||||
tags: str = "",
|
||||
group: str = "",
|
||||
) -> None:
|
||||
if wandb is None:
|
||||
raise ImportError(
|
||||
"wandb is not installed. "
|
||||
"Install it with: pip install 'openjarvis[eval-wandb]'"
|
||||
)
|
||||
self._project = project
|
||||
self._entity = entity or None
|
||||
self._tags: List[str] = [
|
||||
t.strip() for t in tags.split(",") if t.strip()
|
||||
] if tags else []
|
||||
self._group = group or None
|
||||
self._run: Any = None
|
||||
self._step = 0
|
||||
|
||||
def on_run_start(self, config: RunConfig) -> None:
|
||||
run_config = {
|
||||
"benchmark": config.benchmark,
|
||||
"model": config.model,
|
||||
"backend": config.backend,
|
||||
"max_samples": config.max_samples,
|
||||
"max_workers": config.max_workers,
|
||||
"temperature": config.temperature,
|
||||
"max_tokens": config.max_tokens,
|
||||
"seed": config.seed,
|
||||
}
|
||||
if config.agent_name:
|
||||
run_config["agent_name"] = config.agent_name
|
||||
if config.tools:
|
||||
run_config["tools"] = ",".join(config.tools)
|
||||
if config.engine_key:
|
||||
run_config["engine_key"] = config.engine_key
|
||||
|
||||
self._run = wandb.init(
|
||||
project=self._project,
|
||||
entity=self._entity,
|
||||
tags=self._tags or None,
|
||||
group=self._group,
|
||||
config=run_config,
|
||||
reinit=True,
|
||||
)
|
||||
self._step = 0
|
||||
|
||||
def on_result(self, result: EvalResult, config: RunConfig) -> None:
|
||||
if self._run is None:
|
||||
return
|
||||
self._step += 1
|
||||
log_data: Dict[str, Any] = {
|
||||
"sample/is_correct": 1.0 if result.is_correct else 0.0,
|
||||
"sample/latency_seconds": result.latency_seconds,
|
||||
"sample/prompt_tokens": result.prompt_tokens,
|
||||
"sample/completion_tokens": result.completion_tokens,
|
||||
"sample/cost_usd": result.cost_usd,
|
||||
"sample/ttft": result.ttft,
|
||||
"sample/energy_joules": result.energy_joules,
|
||||
"sample/power_watts": result.power_watts,
|
||||
"sample/throughput_tok_per_sec": result.throughput_tok_per_sec,
|
||||
"sample/ipw": result.ipw,
|
||||
"sample/ipj": result.ipj,
|
||||
}
|
||||
if result.error:
|
||||
log_data["sample/has_error"] = 1.0
|
||||
wandb.log(log_data, step=self._step)
|
||||
|
||||
def on_summary(self, summary: RunSummary) -> None:
|
||||
if self._run is None:
|
||||
return
|
||||
flat: Dict[str, Any] = {
|
||||
"accuracy": summary.accuracy,
|
||||
"total_samples": summary.total_samples,
|
||||
"scored_samples": summary.scored_samples,
|
||||
"correct": summary.correct,
|
||||
"errors": summary.errors,
|
||||
"mean_latency_seconds": summary.mean_latency_seconds,
|
||||
"total_cost_usd": summary.total_cost_usd,
|
||||
"total_energy_joules": summary.total_energy_joules,
|
||||
"avg_power_watts": summary.avg_power_watts,
|
||||
"total_input_tokens": summary.total_input_tokens,
|
||||
"total_output_tokens": summary.total_output_tokens,
|
||||
}
|
||||
flat.update(_flatten_metric_stats("accuracy", summary.accuracy_stats))
|
||||
flat.update(_flatten_metric_stats("latency", summary.latency_stats))
|
||||
flat.update(_flatten_metric_stats("ttft", summary.ttft_stats))
|
||||
flat.update(_flatten_metric_stats("energy", summary.energy_stats))
|
||||
flat.update(_flatten_metric_stats("power", summary.power_stats))
|
||||
flat.update(
|
||||
_flatten_metric_stats("gpu_utilization", summary.gpu_utilization_stats)
|
||||
)
|
||||
flat.update(_flatten_metric_stats("throughput", summary.throughput_stats))
|
||||
flat.update(_flatten_metric_stats("mfu", summary.mfu_stats))
|
||||
flat.update(_flatten_metric_stats("mbu", summary.mbu_stats))
|
||||
flat.update(_flatten_metric_stats("ipw", summary.ipw_stats))
|
||||
flat.update(_flatten_metric_stats("ipj", summary.ipj_stats))
|
||||
flat.update(_flatten_metric_stats(
|
||||
"energy_per_output_token",
|
||||
summary.energy_per_output_token_stats,
|
||||
))
|
||||
flat.update(_flatten_metric_stats(
|
||||
"throughput_per_watt",
|
||||
summary.throughput_per_watt_stats,
|
||||
))
|
||||
flat.update(_flatten_metric_stats("itl", summary.itl_stats))
|
||||
wandb.run.summary.update(flat)
|
||||
|
||||
def on_run_end(self) -> None:
|
||||
if self._run is not None:
|
||||
self._run.finish()
|
||||
self._run = None
|
||||
|
||||
|
||||
__all__ = ["WandbTracker"]
|
||||
@@ -0,0 +1,333 @@
|
||||
"""Tests for eval result trackers (W&B + Google Sheets)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from typing import List
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from openjarvis.evals.core.tracker import ResultTracker
|
||||
from openjarvis.evals.core.types import EvalResult, RunConfig, RunSummary
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test double
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class RecordingTracker(ResultTracker):
|
||||
"""Records all lifecycle calls for testing."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.calls: List[str] = []
|
||||
self.results: List[EvalResult] = []
|
||||
self.summary: RunSummary | None = None
|
||||
|
||||
def on_run_start(self, config: RunConfig) -> None:
|
||||
self.calls.append("on_run_start")
|
||||
|
||||
def on_result(self, result: EvalResult, config: RunConfig) -> None:
|
||||
self.calls.append("on_result")
|
||||
self.results.append(result)
|
||||
|
||||
def on_summary(self, summary: RunSummary) -> None:
|
||||
self.calls.append("on_summary")
|
||||
self.summary = summary
|
||||
|
||||
def on_run_end(self) -> None:
|
||||
self.calls.append("on_run_end")
|
||||
|
||||
|
||||
class CrashingTracker(ResultTracker):
|
||||
"""Raises on every lifecycle call."""
|
||||
|
||||
def on_run_start(self, config: RunConfig) -> None:
|
||||
raise RuntimeError("boom start")
|
||||
|
||||
def on_result(self, result: EvalResult, config: RunConfig) -> None:
|
||||
raise RuntimeError("boom result")
|
||||
|
||||
def on_summary(self, summary: RunSummary) -> None:
|
||||
raise RuntimeError("boom summary")
|
||||
|
||||
def on_run_end(self) -> None:
|
||||
raise RuntimeError("boom end")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _make_config(**overrides) -> RunConfig:
|
||||
defaults = dict(benchmark="test", backend="jarvis-direct", model="test-model")
|
||||
defaults.update(overrides)
|
||||
return RunConfig(**defaults)
|
||||
|
||||
|
||||
def _make_summary(**overrides) -> RunSummary:
|
||||
defaults = dict(
|
||||
benchmark="test",
|
||||
category="chat",
|
||||
backend="jarvis-direct",
|
||||
model="test-model",
|
||||
total_samples=10,
|
||||
scored_samples=10,
|
||||
correct=8,
|
||||
accuracy=0.8,
|
||||
errors=0,
|
||||
mean_latency_seconds=1.0,
|
||||
total_cost_usd=0.01,
|
||||
)
|
||||
defaults.update(overrides)
|
||||
return RunSummary(**defaults)
|
||||
|
||||
|
||||
def _make_result(**overrides) -> EvalResult:
|
||||
defaults = dict(record_id="r1", model_answer="answer", is_correct=True)
|
||||
defaults.update(overrides)
|
||||
return EvalResult(**defaults)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# RecordingTracker through EvalRunner lifecycle
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestRecordingTrackerIntegration:
|
||||
"""Test that trackers receive all lifecycle calls through EvalRunner."""
|
||||
|
||||
def test_tracker_lifecycle(self, tmp_path):
|
||||
"""RecordingTracker receives start, result, summary, end calls."""
|
||||
from openjarvis.evals.core.runner import EvalRunner
|
||||
|
||||
# Minimal stubs
|
||||
dataset = MagicMock()
|
||||
record = MagicMock()
|
||||
record.record_id = "r1"
|
||||
record.problem = "What is 1+1?"
|
||||
record.reference = "2"
|
||||
record.category = "chat"
|
||||
record.subject = ""
|
||||
dataset.load = MagicMock()
|
||||
dataset.iter_records = MagicMock(return_value=[record])
|
||||
|
||||
backend = MagicMock()
|
||||
backend.generate_full = MagicMock(return_value={
|
||||
"content": "2",
|
||||
"usage": {"prompt_tokens": 10, "completion_tokens": 5},
|
||||
"latency_seconds": 0.5,
|
||||
})
|
||||
|
||||
scorer = MagicMock()
|
||||
scorer.score = MagicMock(return_value=(True, {}))
|
||||
|
||||
tracker = RecordingTracker()
|
||||
config = _make_config(output_path=str(tmp_path / "out.jsonl"))
|
||||
|
||||
runner = EvalRunner(config, dataset, backend, scorer, trackers=[tracker])
|
||||
runner.run()
|
||||
|
||||
assert "on_run_start" in tracker.calls
|
||||
assert "on_result" in tracker.calls
|
||||
assert "on_summary" in tracker.calls
|
||||
assert "on_run_end" in tracker.calls
|
||||
# Order matters
|
||||
assert tracker.calls.index("on_run_start") < tracker.calls.index("on_result")
|
||||
assert tracker.calls.index("on_result") < tracker.calls.index("on_summary")
|
||||
assert tracker.calls.index("on_summary") < tracker.calls.index("on_run_end")
|
||||
assert len(tracker.results) == 1
|
||||
assert tracker.summary is not None
|
||||
|
||||
def test_crashing_tracker_does_not_abort(self, tmp_path):
|
||||
"""A tracker that raises exceptions must not prevent JSONL output."""
|
||||
from openjarvis.evals.core.runner import EvalRunner
|
||||
|
||||
dataset = MagicMock()
|
||||
record = MagicMock()
|
||||
record.record_id = "r1"
|
||||
record.problem = "What?"
|
||||
record.reference = "yes"
|
||||
record.category = "chat"
|
||||
record.subject = ""
|
||||
dataset.load = MagicMock()
|
||||
dataset.iter_records = MagicMock(return_value=[record])
|
||||
|
||||
backend = MagicMock()
|
||||
backend.generate_full = MagicMock(return_value={
|
||||
"content": "yes",
|
||||
"usage": {},
|
||||
"latency_seconds": 0.1,
|
||||
})
|
||||
|
||||
scorer = MagicMock()
|
||||
scorer.score = MagicMock(return_value=(True, {}))
|
||||
|
||||
output = tmp_path / "out.jsonl"
|
||||
config = _make_config(output_path=str(output))
|
||||
|
||||
crasher = CrashingTracker()
|
||||
runner = EvalRunner(config, dataset, backend, scorer, trackers=[crasher])
|
||||
summary = runner.run()
|
||||
|
||||
# Run completed, JSONL written despite crashing tracker
|
||||
assert summary.total_samples == 1
|
||||
assert output.exists()
|
||||
assert output.read_text().strip() != ""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# WandbTracker unit tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestWandbTracker:
|
||||
"""Unit tests for WandbTracker (mocked wandb module)."""
|
||||
|
||||
def test_import_error_when_wandb_missing(self):
|
||||
"""WandbTracker raises ImportError when wandb is not installed."""
|
||||
with patch.dict(sys.modules, {"wandb": None}):
|
||||
import openjarvis.evals.trackers.wandb_tracker as wt_mod
|
||||
original = wt_mod.wandb
|
||||
wt_mod.wandb = None
|
||||
try:
|
||||
with pytest.raises(ImportError, match="wandb is not installed"):
|
||||
wt_mod.WandbTracker(project="test")
|
||||
finally:
|
||||
wt_mod.wandb = original
|
||||
|
||||
def test_on_result_calls_wandb_log(self):
|
||||
"""on_result calls wandb.log with sample/ prefixed keys."""
|
||||
import openjarvis.evals.trackers.wandb_tracker as wt_mod
|
||||
|
||||
mock_wandb = MagicMock()
|
||||
mock_run = MagicMock()
|
||||
mock_wandb.init = MagicMock(return_value=mock_run)
|
||||
original = wt_mod.wandb
|
||||
wt_mod.wandb = mock_wandb
|
||||
try:
|
||||
tracker = wt_mod.WandbTracker(project="test-proj")
|
||||
config = _make_config()
|
||||
tracker.on_run_start(config)
|
||||
|
||||
result = _make_result(latency_seconds=0.5, energy_joules=1.0)
|
||||
tracker.on_result(result, config)
|
||||
|
||||
mock_wandb.log.assert_called_once()
|
||||
call_args = mock_wandb.log.call_args
|
||||
log_data = call_args[0][0]
|
||||
assert "sample/is_correct" in log_data
|
||||
assert "sample/latency_seconds" in log_data
|
||||
assert log_data["sample/is_correct"] == 1.0
|
||||
assert call_args[1]["step"] == 1
|
||||
|
||||
tracker.on_run_end()
|
||||
finally:
|
||||
wt_mod.wandb = original
|
||||
|
||||
def test_on_summary_updates_run_summary(self):
|
||||
"""on_summary calls wandb.run.summary.update with flat dict."""
|
||||
import openjarvis.evals.trackers.wandb_tracker as wt_mod
|
||||
|
||||
mock_wandb = MagicMock()
|
||||
mock_run = MagicMock()
|
||||
mock_wandb.init = MagicMock(return_value=mock_run)
|
||||
mock_wandb.run = mock_run
|
||||
original = wt_mod.wandb
|
||||
wt_mod.wandb = mock_wandb
|
||||
try:
|
||||
tracker = wt_mod.WandbTracker(project="test-proj")
|
||||
config = _make_config()
|
||||
tracker.on_run_start(config)
|
||||
|
||||
summary = _make_summary()
|
||||
tracker.on_summary(summary)
|
||||
|
||||
mock_run.summary.update.assert_called_once()
|
||||
flat = mock_run.summary.update.call_args[0][0]
|
||||
assert flat["accuracy"] == 0.8
|
||||
assert flat["total_samples"] == 10
|
||||
|
||||
tracker.on_run_end()
|
||||
finally:
|
||||
wt_mod.wandb = original
|
||||
|
||||
def test_reinit_true_for_suite_mode(self):
|
||||
"""wandb.init is called with reinit=True."""
|
||||
import openjarvis.evals.trackers.wandb_tracker as wt_mod
|
||||
|
||||
mock_wandb = MagicMock()
|
||||
mock_run = MagicMock()
|
||||
mock_wandb.init = MagicMock(return_value=mock_run)
|
||||
original = wt_mod.wandb
|
||||
wt_mod.wandb = mock_wandb
|
||||
try:
|
||||
tracker = wt_mod.WandbTracker(project="test-proj", entity="team")
|
||||
config = _make_config()
|
||||
tracker.on_run_start(config)
|
||||
|
||||
call_kwargs = mock_wandb.init.call_args[1]
|
||||
assert call_kwargs["reinit"] is True
|
||||
assert call_kwargs["project"] == "test-proj"
|
||||
assert call_kwargs["entity"] == "team"
|
||||
|
||||
tracker.on_run_end()
|
||||
finally:
|
||||
wt_mod.wandb = original
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# SheetsTracker unit tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestSheetsTracker:
|
||||
"""Unit tests for SheetsTracker."""
|
||||
|
||||
def test_import_error_when_gspread_missing(self):
|
||||
"""SheetsTracker raises ImportError when gspread not installed."""
|
||||
import openjarvis.evals.trackers.sheets_tracker as st_mod
|
||||
original = st_mod.gspread
|
||||
st_mod.gspread = None
|
||||
try:
|
||||
with pytest.raises(ImportError, match="gspread is not installed"):
|
||||
st_mod.SheetsTracker(spreadsheet_id="abc123")
|
||||
finally:
|
||||
st_mod.gspread = original
|
||||
|
||||
def test_on_result_is_noop(self):
|
||||
"""on_result does nothing (no API calls for individual samples)."""
|
||||
import openjarvis.evals.trackers.sheets_tracker as st_mod
|
||||
|
||||
mock_gspread = MagicMock()
|
||||
original = st_mod.gspread
|
||||
st_mod.gspread = mock_gspread
|
||||
original_creds = st_mod.Credentials
|
||||
st_mod.Credentials = MagicMock()
|
||||
try:
|
||||
tracker = st_mod.SheetsTracker(spreadsheet_id="abc123")
|
||||
result = _make_result()
|
||||
config = _make_config()
|
||||
|
||||
# on_result should not call any external API
|
||||
tracker.on_result(result, config)
|
||||
mock_gspread.authorize.assert_not_called()
|
||||
finally:
|
||||
st_mod.gspread = original
|
||||
st_mod.Credentials = original_creds
|
||||
|
||||
def test_build_row_matches_columns(self):
|
||||
"""_build_row returns a list matching SHEET_COLUMNS length."""
|
||||
import openjarvis.evals.trackers.sheets_tracker as st_mod
|
||||
|
||||
mock_gspread = MagicMock()
|
||||
original = st_mod.gspread
|
||||
st_mod.gspread = mock_gspread
|
||||
original_creds = st_mod.Credentials
|
||||
st_mod.Credentials = MagicMock()
|
||||
try:
|
||||
tracker = st_mod.SheetsTracker(spreadsheet_id="abc123")
|
||||
summary = _make_summary()
|
||||
row = tracker._build_row(summary)
|
||||
assert len(row) == len(st_mod.SHEET_COLUMNS), (
|
||||
f"Row length {len(row)} != columns length {len(st_mod.SHEET_COLUMNS)}"
|
||||
)
|
||||
finally:
|
||||
st_mod.gspread = original
|
||||
st_mod.Credentials = original_creds
|
||||
@@ -4,17 +4,22 @@ requires-python = ">=3.10"
|
||||
resolution-markers = [
|
||||
"python_full_version >= '3.14' and sys_platform == 'win32'",
|
||||
"python_full_version >= '3.14' and sys_platform == 'emscripten'",
|
||||
"python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'",
|
||||
"python_full_version >= '3.14' and sys_platform == 'linux'",
|
||||
"python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'win32'",
|
||||
"python_full_version == '3.12.*' and sys_platform == 'win32'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'emscripten'",
|
||||
"python_full_version == '3.12.*' and sys_platform == 'emscripten'",
|
||||
"python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32'",
|
||||
"python_full_version == '3.12.*' and sys_platform != 'emscripten' and sys_platform != 'win32'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'linux'",
|
||||
"python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'",
|
||||
"python_full_version == '3.12.*' and sys_platform == 'linux'",
|
||||
"python_full_version == '3.12.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'",
|
||||
"python_full_version == '3.11.*' and sys_platform == 'win32'",
|
||||
"python_full_version == '3.11.*' and sys_platform == 'emscripten'",
|
||||
"python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'win32'",
|
||||
"python_full_version < '3.11'",
|
||||
"python_full_version == '3.11.*' and sys_platform == 'linux'",
|
||||
"python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'",
|
||||
"python_full_version < '3.11' and sys_platform == 'linux'",
|
||||
"python_full_version < '3.11' and sys_platform != 'linux'",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1078,7 +1083,7 @@ name = "cuda-bindings"
|
||||
version = "12.9.4"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "cuda-pathfinder", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" },
|
||||
{ name = "cuda-pathfinder", marker = "sys_platform == 'linux'" },
|
||||
]
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/7a/d8/b546104b8da3f562c1ff8ab36d130c8fe1dd6a045ced80b4f6ad74f7d4e1/cuda_bindings-12.9.4-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4d3c842c2a4303b2a580fe955018e31aea30278be19795ae05226235268032e5", size = 12148218, upload-time = "2025-10-21T14:51:28.855Z" },
|
||||
@@ -1658,6 +1663,19 @@ requests = [
|
||||
{ name = "requests" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "google-auth-oauthlib"
|
||||
version = "1.3.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "google-auth" },
|
||||
{ name = "requests-oauthlib" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/ac/b4/1b19567e4c567b796f5c593d89895f3cfae5a38e04f27c6af87618fd0942/google_auth_oauthlib-1.3.0.tar.gz", hash = "sha256:cd39e807ac7229d6b8b9c1e297321d36fcc8a9e4857dff4301870985df51a528", size = 21777, upload-time = "2026-02-27T14:13:01.489Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/2f/56/909fd5632226d3fba31d7aeffd4754410735d49362f5809956fe3e9af344/google_auth_oauthlib-1.3.0-py3-none-any.whl", hash = "sha256:386b3fb85cf4a5b819c6ad23e3128d975216b4cac76324de1d90b128aaf38f29", size = 19308, upload-time = "2026-02-27T14:12:47.865Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "google-genai"
|
||||
version = "1.64.0"
|
||||
@@ -1821,6 +1839,19 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/1c/ce/adfe7e5f701d503be7778291757452e3fab6b19acf51917c79f5d1cf7f8a/grpcio-1.78.1-cp314-cp314-win_amd64.whl", hash = "sha256:e2a6b33d1050dce2c6f563c5caf7f7cbeebf7fba8cde37ffe3803d50526900d1", size = 4932000, upload-time = "2026-02-20T01:15:36.127Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "gspread"
|
||||
version = "6.2.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "google-auth" },
|
||||
{ name = "google-auth-oauthlib" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/91/83/42d1d813822ed016d77aabadc99b09de3b5bd68532fd6bae23fd62347c41/gspread-6.2.1.tar.gz", hash = "sha256:2c7c99f7c32ebea6ec0d36f2d5cbe8a2be5e8f2a48bde87ad1ea203eff32bd03", size = 82590, upload-time = "2025-05-14T15:56:25.254Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/27/76/563fb20dedd0e12794d9a12cfe0198458cc0501fdc7b034eee2166d035d5/gspread-6.2.1-py3-none-any.whl", hash = "sha256:6d4ec9f1c23ae3c704a9219026dac01f2b328ac70b96f1495055d453c4c184db", size = 59977, upload-time = "2025-05-14T15:56:24.014Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "h11"
|
||||
version = "0.16.0"
|
||||
@@ -2814,7 +2845,8 @@ name = "networkx"
|
||||
version = "3.4.2"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
resolution-markers = [
|
||||
"python_full_version < '3.11'",
|
||||
"python_full_version < '3.11' and sys_platform == 'linux'",
|
||||
"python_full_version < '3.11' and sys_platform != 'linux'",
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/fd/1d/06475e1cd5264c0b870ea2cc6fdb3e37177c1e565c43f56ff17a10e3937f/networkx-3.4.2.tar.gz", hash = "sha256:307c3669428c5362aab27c8a1260aa8f47c4e91d3891f48be0141738d8d053e1", size = 2151368, upload-time = "2024-10-21T12:39:38.695Z" }
|
||||
wheels = [
|
||||
@@ -2828,16 +2860,20 @@ source = { registry = "https://pypi.org/simple" }
|
||||
resolution-markers = [
|
||||
"python_full_version >= '3.14' and sys_platform == 'win32'",
|
||||
"python_full_version >= '3.14' and sys_platform == 'emscripten'",
|
||||
"python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'",
|
||||
"python_full_version >= '3.14' and sys_platform == 'linux'",
|
||||
"python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'win32'",
|
||||
"python_full_version == '3.12.*' and sys_platform == 'win32'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'emscripten'",
|
||||
"python_full_version == '3.12.*' and sys_platform == 'emscripten'",
|
||||
"python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32'",
|
||||
"python_full_version == '3.12.*' and sys_platform != 'emscripten' and sys_platform != 'win32'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'linux'",
|
||||
"python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'",
|
||||
"python_full_version == '3.12.*' and sys_platform == 'linux'",
|
||||
"python_full_version == '3.12.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'",
|
||||
"python_full_version == '3.11.*' and sys_platform == 'win32'",
|
||||
"python_full_version == '3.11.*' and sys_platform == 'emscripten'",
|
||||
"python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'win32'",
|
||||
"python_full_version == '3.11.*' and sys_platform == 'linux'",
|
||||
"python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'",
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/6a/51/63fe664f3908c97be9d2e4f1158eb633317598cfa6e1fc14af5383f17512/networkx-3.6.1.tar.gz", hash = "sha256:26b7c357accc0c8cde558ad486283728b65b6a95d85ee1cd66bafab4c8168509", size = 2517025, upload-time = "2025-12-08T17:02:39.908Z" }
|
||||
wheels = [
|
||||
@@ -2875,7 +2911,8 @@ name = "numpy"
|
||||
version = "2.2.6"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
resolution-markers = [
|
||||
"python_full_version < '3.11'",
|
||||
"python_full_version < '3.11' and sys_platform == 'linux'",
|
||||
"python_full_version < '3.11' and sys_platform != 'linux'",
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/76/21/7d2a95e4bba9dc13d043ee156a356c0a8f0c6309dff6b21b4d71a073b8a8/numpy-2.2.6.tar.gz", hash = "sha256:e29554e2bef54a90aa5cc07da6ce955accb83f21ab5de01a62c8478897b264fd", size = 20276440, upload-time = "2025-05-17T22:38:04.611Z" }
|
||||
wheels = [
|
||||
@@ -2942,16 +2979,20 @@ source = { registry = "https://pypi.org/simple" }
|
||||
resolution-markers = [
|
||||
"python_full_version >= '3.14' and sys_platform == 'win32'",
|
||||
"python_full_version >= '3.14' and sys_platform == 'emscripten'",
|
||||
"python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'",
|
||||
"python_full_version >= '3.14' and sys_platform == 'linux'",
|
||||
"python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'win32'",
|
||||
"python_full_version == '3.12.*' and sys_platform == 'win32'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'emscripten'",
|
||||
"python_full_version == '3.12.*' and sys_platform == 'emscripten'",
|
||||
"python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32'",
|
||||
"python_full_version == '3.12.*' and sys_platform != 'emscripten' and sys_platform != 'win32'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'linux'",
|
||||
"python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'",
|
||||
"python_full_version == '3.12.*' and sys_platform == 'linux'",
|
||||
"python_full_version == '3.12.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'",
|
||||
"python_full_version == '3.11.*' and sys_platform == 'win32'",
|
||||
"python_full_version == '3.11.*' and sys_platform == 'emscripten'",
|
||||
"python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'win32'",
|
||||
"python_full_version == '3.11.*' and sys_platform == 'linux'",
|
||||
"python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'",
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/57/fd/0005efbd0af48e55eb3c7208af93f2862d4b1a56cd78e84309a2d959208d/numpy-2.4.2.tar.gz", hash = "sha256:659a6107e31a83c4e33f763942275fd278b21d095094044eb35569e86a21ddae", size = 20723651, upload-time = "2026-01-31T23:13:10.135Z" }
|
||||
wheels = [
|
||||
@@ -3065,7 +3106,7 @@ name = "nvidia-cudnn-cu12"
|
||||
version = "9.10.2.21"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "nvidia-cublas-cu12", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" },
|
||||
{ name = "nvidia-cublas-cu12", marker = "sys_platform == 'linux'" },
|
||||
]
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/ba/51/e123d997aa098c61d029f76663dedbfb9bc8dcf8c60cbd6adbe42f76d049/nvidia_cudnn_cu12-9.10.2.21-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:949452be657fa16687d0930933f032835951ef0892b37d2d53824d1a84dc97a8", size = 706758467, upload-time = "2025-06-06T21:54:08.597Z" },
|
||||
@@ -3076,7 +3117,7 @@ name = "nvidia-cufft-cu12"
|
||||
version = "11.3.3.83"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "nvidia-nvjitlink-cu12", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" },
|
||||
{ name = "nvidia-nvjitlink-cu12", marker = "sys_platform == 'linux'" },
|
||||
]
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/1f/13/ee4e00f30e676b66ae65b4f08cb5bcbb8392c03f54f2d5413ea99a5d1c80/nvidia_cufft_cu12-11.3.3.83-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4d2dd21ec0b88cf61b62e6b43564355e5222e4a3fb394cac0db101f2dd0d4f74", size = 193118695, upload-time = "2025-03-07T01:45:27.821Z" },
|
||||
@@ -3103,9 +3144,9 @@ name = "nvidia-cusolver-cu12"
|
||||
version = "11.7.3.90"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "nvidia-cublas-cu12", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" },
|
||||
{ name = "nvidia-cusparse-cu12", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" },
|
||||
{ name = "nvidia-nvjitlink-cu12", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" },
|
||||
{ name = "nvidia-cublas-cu12", marker = "sys_platform == 'linux'" },
|
||||
{ name = "nvidia-cusparse-cu12", marker = "sys_platform == 'linux'" },
|
||||
{ name = "nvidia-nvjitlink-cu12", marker = "sys_platform == 'linux'" },
|
||||
]
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/85/48/9a13d2975803e8cf2777d5ed57b87a0b6ca2cc795f9a4f59796a910bfb80/nvidia_cusolver_cu12-11.7.3.90-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:4376c11ad263152bd50ea295c05370360776f8c3427b30991df774f9fb26c450", size = 267506905, upload-time = "2025-03-07T01:47:16.273Z" },
|
||||
@@ -3116,7 +3157,7 @@ name = "nvidia-cusparse-cu12"
|
||||
version = "12.5.8.93"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "nvidia-nvjitlink-cu12", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" },
|
||||
{ name = "nvidia-nvjitlink-cu12", marker = "sys_platform == 'linux'" },
|
||||
]
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/c2/f5/e1854cb2f2bcd4280c44736c93550cc300ff4b8c95ebe370d0aa7d2b473d/nvidia_cusparse_cu12-12.5.8.93-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1ec05d76bbbd8b61b06a80e1eaf8cf4959c3d4ce8e711b65ebd0443bb0ebb13b", size = 288216466, upload-time = "2025-03-07T01:48:13.779Z" },
|
||||
@@ -3171,6 +3212,15 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/a2/eb/86626c1bbc2edb86323022371c39aa48df6fd8b0a1647bc274577f72e90b/nvidia_nvtx_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5b17e2001cc0d751a5bc2c6ec6d26ad95913324a4adb86788c944f8ce9ba441f", size = 89954, upload-time = "2025-03-07T01:42:44.131Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "oauthlib"
|
||||
version = "3.3.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/0b/5f/19930f824ffeb0ad4372da4812c50edbd1434f678c90c2733e1188edfc63/oauthlib-3.3.1.tar.gz", hash = "sha256:0f0f8aa759826a193cf66c12ea1af1637f87b9b4622d46e866952bb022e538c9", size = 185918, upload-time = "2025-06-19T22:48:08.269Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/be/9c/92789c596b8df838baa98fa71844d84283302f7604ed565dafe5a6b5041a/oauthlib-3.3.1-py3-none-any.whl", hash = "sha256:88119c938d2b8fb88561af5f6ee0eec8cc8d552b7bb1f712743136eb7523b7a1", size = 160065, upload-time = "2025-06-19T22:48:06.508Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "onnxruntime"
|
||||
version = "1.24.2"
|
||||
@@ -3346,6 +3396,13 @@ energy-amd = [
|
||||
energy-apple = [
|
||||
{ name = "zeus-ml" },
|
||||
]
|
||||
eval-sheets = [
|
||||
{ name = "google-auth" },
|
||||
{ name = "gspread" },
|
||||
]
|
||||
eval-wandb = [
|
||||
{ name = "wandb" },
|
||||
]
|
||||
gpu-metrics = [
|
||||
{ name = "pynvml" },
|
||||
]
|
||||
@@ -3429,7 +3486,9 @@ requires-dist = [
|
||||
{ name = "faiss-cpu", marker = "extra == 'memory-faiss'", specifier = ">=1.7" },
|
||||
{ name = "fastapi", marker = "extra == 'server'", specifier = ">=0.110" },
|
||||
{ name = "faster-whisper", marker = "extra == 'speech'", specifier = ">=1.0" },
|
||||
{ name = "google-auth", marker = "extra == 'eval-sheets'", specifier = ">=2.0" },
|
||||
{ name = "google-genai", marker = "extra == 'inference-google'", specifier = ">=1.0" },
|
||||
{ name = "gspread", marker = "extra == 'eval-sheets'", specifier = ">=6.0" },
|
||||
{ name = "httpx", specifier = ">=0.27" },
|
||||
{ name = "line-bot-sdk", marker = "extra == 'channel-line'", specifier = ">=3.0" },
|
||||
{ name = "litellm", marker = "extra == 'inference-litellm'", specifier = ">=1.40" },
|
||||
@@ -3473,12 +3532,13 @@ requires-dist = [
|
||||
{ name = "twitchio", marker = "extra == 'channel-twitch'", specifier = ">=2.6" },
|
||||
{ name = "uvicorn", marker = "extra == 'server'", specifier = ">=0.30" },
|
||||
{ name = "viberbot", marker = "extra == 'channel-viber'", specifier = ">=1.0" },
|
||||
{ name = "wandb", marker = "extra == 'eval-wandb'", specifier = ">=0.17" },
|
||||
{ name = "wasmtime", marker = "extra == 'sandbox-wasm'", specifier = ">=25" },
|
||||
{ name = "zeus-ml", extras = ["apple"], marker = "extra == 'energy-all'" },
|
||||
{ name = "zeus-ml", extras = ["apple"], marker = "extra == 'energy-apple'" },
|
||||
{ name = "zulip", marker = "extra == 'channel-zulip'", specifier = ">=0.9" },
|
||||
]
|
||||
provides-extras = ["dev", "inference-ollama", "inference-vllm", "inference-llamacpp", "inference-mlx", "inference-cloud", "inference-google", "inference-litellm", "tools-search", "memory-faiss", "memory-colbert", "memory-pdf", "memory-bm25", "server", "agents", "openhands", "claude-code", "gpu-metrics", "energy-amd", "energy-apple", "energy-all", "learning", "orchestrator-training", "channel-telegram", "channel-discord", "channel-slack", "channel-webhook", "channel-email", "channel-whatsapp", "channel-signal", "channel-google-chat", "channel-irc", "channel-webchat", "channel-teams", "channel-matrix", "channel-mattermost", "channel-feishu", "channel-bluebubbles", "channel-whatsapp-baileys", "channel-line", "channel-viber", "channel-messenger", "channel-reddit", "channel-mastodon", "channel-xmpp", "channel-rocketchat", "channel-zulip", "channel-twitch", "channel-nostr", "browser", "media", "pdf", "scheduler", "security-signing", "sandbox-wasm", "dashboard", "speech", "speech-deepgram", "docs"]
|
||||
provides-extras = ["dev", "inference-mlx", "inference-cloud", "inference-google", "inference-litellm", "tools-search", "memory-faiss", "memory-colbert", "memory-pdf", "memory-bm25", "server", "openhands", "gpu-metrics", "energy-amd", "energy-apple", "energy-all", "orchestrator-training", "channel-telegram", "channel-discord", "channel-slack", "channel-line", "channel-viber", "channel-messenger", "channel-reddit", "channel-mastodon", "channel-xmpp", "channel-rocketchat", "channel-zulip", "channel-twitch", "channel-nostr", "browser", "media", "pdf", "scheduler", "security-signing", "sandbox-wasm", "dashboard", "speech", "speech-deepgram", "eval-wandb", "eval-sheets", "docs"]
|
||||
|
||||
[[package]]
|
||||
name = "opentelemetry-api"
|
||||
@@ -3722,7 +3782,8 @@ name = "pandas"
|
||||
version = "2.3.3"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
resolution-markers = [
|
||||
"python_full_version < '3.11'",
|
||||
"python_full_version < '3.11' and sys_platform == 'linux'",
|
||||
"python_full_version < '3.11' and sys_platform != 'linux'",
|
||||
]
|
||||
dependencies = [
|
||||
{ name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
|
||||
@@ -3788,16 +3849,20 @@ source = { registry = "https://pypi.org/simple" }
|
||||
resolution-markers = [
|
||||
"python_full_version >= '3.14' and sys_platform == 'win32'",
|
||||
"python_full_version >= '3.14' and sys_platform == 'emscripten'",
|
||||
"python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'",
|
||||
"python_full_version >= '3.14' and sys_platform == 'linux'",
|
||||
"python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'win32'",
|
||||
"python_full_version == '3.12.*' and sys_platform == 'win32'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'emscripten'",
|
||||
"python_full_version == '3.12.*' and sys_platform == 'emscripten'",
|
||||
"python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32'",
|
||||
"python_full_version == '3.12.*' and sys_platform != 'emscripten' and sys_platform != 'win32'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'linux'",
|
||||
"python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'",
|
||||
"python_full_version == '3.12.*' and sys_platform == 'linux'",
|
||||
"python_full_version == '3.12.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'",
|
||||
"python_full_version == '3.11.*' and sys_platform == 'win32'",
|
||||
"python_full_version == '3.11.*' and sys_platform == 'emscripten'",
|
||||
"python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'win32'",
|
||||
"python_full_version == '3.11.*' and sys_platform == 'linux'",
|
||||
"python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'",
|
||||
]
|
||||
dependencies = [
|
||||
{ name = "numpy", version = "2.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
|
||||
@@ -5071,6 +5136,19 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/1e/db/4254e3eabe8020b458f1a747140d32277ec7a271daf1d235b70dc0b4e6e3/requests-2.32.5-py3-none-any.whl", hash = "sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6", size = 64738, upload-time = "2025-08-18T20:46:00.542Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "requests-oauthlib"
|
||||
version = "2.0.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "oauthlib" },
|
||||
{ name = "requests" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/42/f2/05f29bc3913aea15eb670be136045bf5c5bbf4b99ecb839da9b422bb2c85/requests-oauthlib-2.0.0.tar.gz", hash = "sha256:b3dffaebd884d8cd778494369603a9e7b58d29111bf6b41bdc2dcd87203af4e9", size = 55650, upload-time = "2024-03-22T20:32:29.939Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/3b/5d/63d4ae3b9daea098d5d6f5da83984853c1bbacd5dc826764b249fe119d24/requests_oauthlib-2.0.0-py2.py3-none-any.whl", hash = "sha256:7dd8a5c40426b779b0868c404bdef9768deccf22749cde15852df527e6269b36", size = 24179, upload-time = "2024-03-22T20:32:28.055Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "requests-toolbelt"
|
||||
version = "1.0.0"
|
||||
@@ -5324,7 +5402,8 @@ name = "scikit-learn"
|
||||
version = "1.7.2"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
resolution-markers = [
|
||||
"python_full_version < '3.11'",
|
||||
"python_full_version < '3.11' and sys_platform == 'linux'",
|
||||
"python_full_version < '3.11' and sys_platform != 'linux'",
|
||||
]
|
||||
dependencies = [
|
||||
{ name = "joblib", marker = "python_full_version < '3.11'" },
|
||||
@@ -5373,16 +5452,20 @@ source = { registry = "https://pypi.org/simple" }
|
||||
resolution-markers = [
|
||||
"python_full_version >= '3.14' and sys_platform == 'win32'",
|
||||
"python_full_version >= '3.14' and sys_platform == 'emscripten'",
|
||||
"python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'",
|
||||
"python_full_version >= '3.14' and sys_platform == 'linux'",
|
||||
"python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'win32'",
|
||||
"python_full_version == '3.12.*' and sys_platform == 'win32'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'emscripten'",
|
||||
"python_full_version == '3.12.*' and sys_platform == 'emscripten'",
|
||||
"python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32'",
|
||||
"python_full_version == '3.12.*' and sys_platform != 'emscripten' and sys_platform != 'win32'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'linux'",
|
||||
"python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'",
|
||||
"python_full_version == '3.12.*' and sys_platform == 'linux'",
|
||||
"python_full_version == '3.12.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'",
|
||||
"python_full_version == '3.11.*' and sys_platform == 'win32'",
|
||||
"python_full_version == '3.11.*' and sys_platform == 'emscripten'",
|
||||
"python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'win32'",
|
||||
"python_full_version == '3.11.*' and sys_platform == 'linux'",
|
||||
"python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'",
|
||||
]
|
||||
dependencies = [
|
||||
{ name = "joblib", marker = "python_full_version >= '3.11'" },
|
||||
@@ -5435,7 +5518,8 @@ name = "scipy"
|
||||
version = "1.15.3"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
resolution-markers = [
|
||||
"python_full_version < '3.11'",
|
||||
"python_full_version < '3.11' and sys_platform == 'linux'",
|
||||
"python_full_version < '3.11' and sys_platform != 'linux'",
|
||||
]
|
||||
dependencies = [
|
||||
{ name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
|
||||
@@ -5496,16 +5580,20 @@ source = { registry = "https://pypi.org/simple" }
|
||||
resolution-markers = [
|
||||
"python_full_version >= '3.14' and sys_platform == 'win32'",
|
||||
"python_full_version >= '3.14' and sys_platform == 'emscripten'",
|
||||
"python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'",
|
||||
"python_full_version >= '3.14' and sys_platform == 'linux'",
|
||||
"python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'win32'",
|
||||
"python_full_version == '3.12.*' and sys_platform == 'win32'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'emscripten'",
|
||||
"python_full_version == '3.12.*' and sys_platform == 'emscripten'",
|
||||
"python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32'",
|
||||
"python_full_version == '3.12.*' and sys_platform != 'emscripten' and sys_platform != 'win32'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'linux'",
|
||||
"python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'",
|
||||
"python_full_version == '3.12.*' and sys_platform == 'linux'",
|
||||
"python_full_version == '3.12.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'",
|
||||
"python_full_version == '3.11.*' and sys_platform == 'win32'",
|
||||
"python_full_version == '3.11.*' and sys_platform == 'emscripten'",
|
||||
"python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'win32'",
|
||||
"python_full_version == '3.11.*' and sys_platform == 'linux'",
|
||||
"python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'",
|
||||
]
|
||||
dependencies = [
|
||||
{ name = "numpy", version = "2.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
|
||||
@@ -5579,8 +5667,8 @@ name = "secretstorage"
|
||||
version = "3.5.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "cryptography", marker = "python_full_version >= '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'" },
|
||||
{ name = "jeepney", marker = "python_full_version >= '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'" },
|
||||
{ name = "cryptography", marker = "python_full_version >= '3.12' and sys_platform == 'linux'" },
|
||||
{ name = "jeepney", marker = "python_full_version >= '3.12' and sys_platform == 'linux'" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/1c/03/e834bcd866f2f8a49a85eaff47340affa3bfa391ee9912a952a1faa68c7b/secretstorage-3.5.0.tar.gz", hash = "sha256:f04b8e4689cbce351744d5537bf6b1329c6fc68f91fa666f60a380edddcd11be", size = 19884, upload-time = "2025-11-23T19:02:53.191Z" }
|
||||
wheels = [
|
||||
@@ -5638,6 +5726,19 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/03/b0/811dae8fb9f2784e138785d481469788f2e0d0c109c5737372454415f55f/sentencepiece-0.2.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:477c81505db072b3ab627e7eab972ea1025331bd3a92bacbf798df2b75ea86ec", size = 1254846, upload-time = "2025-08-12T07:00:40.611Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sentry-sdk"
|
||||
version = "2.54.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "certifi" },
|
||||
{ name = "urllib3" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/c8/e9/2e3a46c304e7fa21eaa70612f60354e32699c7102eb961f67448e222ad7c/sentry_sdk-2.54.0.tar.gz", hash = "sha256:2620c2575128d009b11b20f7feb81e4e4e8ae08ec1d36cbc845705060b45cc1b", size = 413813, upload-time = "2026-03-02T15:12:41.355Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/53/39/be412cc86bc6247b8f69e9383d7950711bd86f8d0a4a4b0fe8fad685bc21/sentry_sdk-2.54.0-py2.py3-none-any.whl", hash = "sha256:fd74e0e281dcda63afff095d23ebcd6e97006102cdc8e78a29f19ecdf796a0de", size = 439198, upload-time = "2026-03-02T15:12:39.546Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "setuptools"
|
||||
version = "82.0.0"
|
||||
@@ -5679,7 +5780,8 @@ name = "slixmpp"
|
||||
version = "1.12.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
resolution-markers = [
|
||||
"python_full_version < '3.11'",
|
||||
"python_full_version < '3.11' and sys_platform == 'linux'",
|
||||
"python_full_version < '3.11' and sys_platform != 'linux'",
|
||||
]
|
||||
dependencies = [
|
||||
{ name = "aiodns", marker = "python_full_version < '3.11'" },
|
||||
@@ -5717,16 +5819,20 @@ source = { registry = "https://pypi.org/simple" }
|
||||
resolution-markers = [
|
||||
"python_full_version >= '3.14' and sys_platform == 'win32'",
|
||||
"python_full_version >= '3.14' and sys_platform == 'emscripten'",
|
||||
"python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'",
|
||||
"python_full_version >= '3.14' and sys_platform == 'linux'",
|
||||
"python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'win32'",
|
||||
"python_full_version == '3.12.*' and sys_platform == 'win32'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'emscripten'",
|
||||
"python_full_version == '3.12.*' and sys_platform == 'emscripten'",
|
||||
"python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32'",
|
||||
"python_full_version == '3.12.*' and sys_platform != 'emscripten' and sys_platform != 'win32'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'linux'",
|
||||
"python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'",
|
||||
"python_full_version == '3.12.*' and sys_platform == 'linux'",
|
||||
"python_full_version == '3.12.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'",
|
||||
"python_full_version == '3.11.*' and sys_platform == 'win32'",
|
||||
"python_full_version == '3.11.*' and sys_platform == 'emscripten'",
|
||||
"python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'win32'",
|
||||
"python_full_version == '3.11.*' and sys_platform == 'linux'",
|
||||
"python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'",
|
||||
]
|
||||
dependencies = [
|
||||
{ name = "aiodns", marker = "python_full_version >= '3.11'" },
|
||||
@@ -6146,7 +6252,8 @@ name = "twitchio"
|
||||
version = "2.10.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
resolution-markers = [
|
||||
"python_full_version < '3.11'",
|
||||
"python_full_version < '3.11' and sys_platform == 'linux'",
|
||||
"python_full_version < '3.11' and sys_platform != 'linux'",
|
||||
]
|
||||
dependencies = [
|
||||
{ name = "aiohttp", marker = "python_full_version < '3.11'" },
|
||||
@@ -6165,16 +6272,20 @@ source = { registry = "https://pypi.org/simple" }
|
||||
resolution-markers = [
|
||||
"python_full_version >= '3.14' and sys_platform == 'win32'",
|
||||
"python_full_version >= '3.14' and sys_platform == 'emscripten'",
|
||||
"python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'",
|
||||
"python_full_version >= '3.14' and sys_platform == 'linux'",
|
||||
"python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'win32'",
|
||||
"python_full_version == '3.12.*' and sys_platform == 'win32'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'emscripten'",
|
||||
"python_full_version == '3.12.*' and sys_platform == 'emscripten'",
|
||||
"python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32'",
|
||||
"python_full_version == '3.12.*' and sys_platform != 'emscripten' and sys_platform != 'win32'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'linux'",
|
||||
"python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'",
|
||||
"python_full_version == '3.12.*' and sys_platform == 'linux'",
|
||||
"python_full_version == '3.12.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'",
|
||||
"python_full_version == '3.11.*' and sys_platform == 'win32'",
|
||||
"python_full_version == '3.11.*' and sys_platform == 'emscripten'",
|
||||
"python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'win32'",
|
||||
"python_full_version == '3.11.*' and sys_platform == 'linux'",
|
||||
"python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'",
|
||||
]
|
||||
dependencies = [
|
||||
{ name = "aiohttp", marker = "python_full_version >= '3.11'" },
|
||||
@@ -6404,6 +6515,35 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/5d/c1/f01846f10c8383b197a67b4cdfea6eea476aab678ed9f6f5d9ccc8c8dddf/viberbot-1.0.12-py3-none-any.whl", hash = "sha256:ca43fea2945d650c2ef2cbd777f3c546c795bf945278f6620ceda42f4101d801", size = 26164, upload-time = "2021-01-27T13:45:43.726Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wandb"
|
||||
version = "0.25.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "click" },
|
||||
{ name = "gitpython" },
|
||||
{ name = "packaging" },
|
||||
{ name = "platformdirs" },
|
||||
{ name = "protobuf" },
|
||||
{ name = "pydantic" },
|
||||
{ name = "pyyaml" },
|
||||
{ name = "requests" },
|
||||
{ name = "sentry-sdk" },
|
||||
{ name = "typing-extensions" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/fd/60/d94952549920469524b689479c864c692ca47eca4b8c2fe3389b64a58778/wandb-0.25.0.tar.gz", hash = "sha256:45840495a288e34245d69d07b5a0b449220fbc5b032e6b51c4f92ec9026d2ad1", size = 43951335, upload-time = "2026-02-13T00:17:45.515Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/c1/7d/0c131db3ec9deaabbd32263d90863cbfbe07659527e11c35a5c738cecdc5/wandb-0.25.0-py3-none-macosx_12_0_arm64.whl", hash = "sha256:5eecb3c7b5e60d1acfa4b056bfbaa0b79a482566a9db58c9f99724b3862bc8e5", size = 23287536, upload-time = "2026-02-13T00:17:20.265Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c3/95/31bb7f76a966ec87495e5a72ac7570685be162494c41757ac871768dbc4f/wandb-0.25.0-py3-none-macosx_12_0_x86_64.whl", hash = "sha256:daeedaadb183dc466e634fba90ab2bab1d4e93000912be0dee95065a0624a3fd", size = 25196062, upload-time = "2026-02-13T00:17:23.356Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d9/a1/258cdedbf30cebc692198a774cf0ef945b7ed98ee64bdaf62621281c95d8/wandb-0.25.0-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:5e0127dbcef13eea48f4b84268da7004d34d3120ebc7b2fa9cefb72b49dbb825", size = 22799744, upload-time = "2026-02-13T00:17:26.437Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/de/91/ec9465d014cfd199c5b2083d271d31b3c2aedeae66f3d8a0712f7f54bdf3/wandb-0.25.0-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:6c4c38077836f9b7569a35b0e1dcf1f0c43616fcd936d182f475edbfea063665", size = 25262839, upload-time = "2026-02-13T00:17:28.8Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c7/95/cb2d1c7143f534544147fb53fe87944508b8cb9a058bc5b6f8a94adbee15/wandb-0.25.0-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:6edd8948d305cb73745bf564b807bd73da2ccbd47c548196b8a362f7df40aed8", size = 22853714, upload-time = "2026-02-13T00:17:31.68Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d7/94/68163f70c1669edcf130822aaaea782d8198b5df74443eca0085ec596774/wandb-0.25.0-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:ada6f08629bb014ad6e0a19d5dec478cdaa116431baa3f0a4bf4ab8d9893611f", size = 25358037, upload-time = "2026-02-13T00:17:34.676Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/cc/fb/9578eed2c01b2fc6c8b693da110aa9c73a33d7bb556480f5cfc42e48c94e/wandb-0.25.0-py3-none-win32.whl", hash = "sha256:020b42ca4d76e347709d65f59b30d4623a115edc28f462af1c92681cb17eae7c", size = 24604118, upload-time = "2026-02-13T00:17:37.641Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/25/97/460f6cb738aaa39b4eb2e6b4c630b2ae4321cdd70a79d5955ea75a878981/wandb-0.25.0-py3-none-win_amd64.whl", hash = "sha256:78307ac0b328f2dc334c8607bec772851215584b62c439eb320c4af4fb077a00", size = 24604122, upload-time = "2026-02-13T00:17:39.991Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/27/6c/5847b4dda1dfd52630dac08711d4348c69ed657f0698fc2d949c7f7a6622/wandb-0.25.0-py3-none-win_arm64.whl", hash = "sha256:c6174401fd6fb726295e98d57b4231c100eca96bd17de51bfc64038a57230aaf", size = 21785298, upload-time = "2026-02-13T00:17:42.475Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasmtime"
|
||||
version = "42.0.0"
|
||||
|
||||
Reference in New Issue
Block a user