Hyperparameter Tuning

Duration: 55 min

Hyperparameter tuning automates the search for optimal model parameters. This module covers HyperparameterTuner, Bayesian optimization, warm start, and best practices for efficient tuning.

What is Hyperparameter Tuning?

Hyperparameter tuning automatically searches for the best combination of hyperparameters to maximize model performance. SageMaker uses Bayesian optimization to intelligently explore the parameter space, reducing the number of training jobs needed.

Creating a HyperparameterTuner

from sagemaker.tuner import HyperparameterTuner, IntegerParameter, ContinuousParameter, CategoricalParameter
from sagemaker.xgboost import XGBoost
import sagemaker

session = sagemaker.Session()
role = 'arn:aws:iam::123456789012:role/SageMakerRole'
bucket = session.default_bucket()

# Create base estimator
xgb_estimator = XGBoost(
    entry_point='train.py',
    role=role,
    instance_count=1,
    instance_type='ml.m5.xlarge',
    framework_version='1.5',
    output_path=f's3://{bucket}/xgb-output',
    sagemaker_session=session
)

# Define hyperparameter ranges
hyperparameter_ranges = {
    'max_depth': IntegerParameter(1, 10),
    'eta': ContinuousParameter(0.1, 0.5),
    'min_child_weight': IntegerParameter(2, 10),
    'subsample': ContinuousParameter(0.5, 1.0),
    'gamma': ContinuousParameter(0, 5)
}

# Create tuner
tuner = HyperparameterTuner(
    estimator=xgb_estimator,
    objective_metric_name='validation:auc',
    hyperparameter_ranges=hyperparameter_ranges,
    metric_definitions=[
        {'Name': 'validation:auc', 'Regex': 'validation-auc=([0-9\\.]+)'}
    ],
    max_jobs=20,
    max_parallel_jobs=4,
    base_tuning_job_name='xgb-tuning'
)

# Start tuning
tuner.fit(
    {'training': f's3://{bucket}/train-data/'},
    job_name='xgb-tuning-job'
)

Bayesian Optimization Strategy

from sagemaker.tuner import HyperparameterTuner, StrategyConfig

# Configure Bayesian optimization
strategy_config = StrategyConfig(
    strategy='Bayesian',
    metric_definitions=[
        {'Name': 'validation:accuracy', 'Regex': 'accuracy=([0-9\\.]+)'}
    ]
)

tuner = HyperparameterTuner(
    estimator=xgb_estimator,
    objective_metric_name='validation:accuracy',
    hyperparameter_ranges=hyperparameter_ranges,
    max_jobs=30,
    max_parallel_jobs=5,
    strategy='Bayesian'
)

# Bayesian optimization learns from previous trials
tuner.fit({'training': f's3://{bucket}/train-data/'})

Warm Start for Tuning

from sagemaker.tuner import WarmStartConfig, WarmStartTypes

# Use results from previous tuning job
warm_start_config = WarmStartConfig(
    type=WarmStartTypes.TRANSFER_LEARNING,
    from_job_name='xgb-tuning-job-2024-01-10'
)

# Create new tuner with warm start
tuner = HyperparameterTuner(
    estimator=xgb_estimator,
    objective_metric_name='validation:auc',
    hyperparameter_ranges=hyperparameter_ranges,
    max_jobs=20,
    max_parallel_jobs=4,
    warm_start_config=warm_start_config
)

tuner.fit({'training': f's3://{bucket}/train-data/'})

Tuning Job Configuration

{
  "tuning_job_config": {
    "tuning_job_name": "xgb-tuning-job",
    "tuning_objective": {
      "metric_name": "validation:auc",
      "type": "Maximize"
    },
    "resource_limits": {
      "max_number_of_training_jobs": 20,
      "max_parallel_training_jobs": 4
    },
    "parameter_ranges": {
      "integer_parameter_ranges": [
        {
          "name": "max_depth",
          "min_value": "1",
          "max_value": "10"
        }
      ],
      "continuous_parameter_ranges": [
        {
          "name": "eta",
          "min_value": "0.1",
          "max_value": "0.5"
        }
      ]
    },
    "strategy_config": {
      "strategy": "Bayesian"
    }
  }
}

Analyzing Tuning Results

# Get best training job
best_job = tuner.best_training_job()
print(f"Best job: {best_job}")

# Get best hyperparameters
best_hyperparameters = tuner.best_estimator().hyperparameters()
print(f"Best hyperparameters: {best_hyperparameters}")

# Deploy best model
best_predictor = tuner.deploy(
    initial_instance_count=1,
    instance_type='ml.m5.large'
)

Early Stopping for Efficiency

from sagemaker.tuner import HyperparameterTuner

tuner = HyperparameterTuner(
    estimator=xgb_estimator,
    objective_metric_name='validation:auc',
    hyperparameter_ranges=hyperparameter_ranges,
    max_jobs=20,
    max_parallel_jobs=4,
    early_stopping_type='Auto'  # Enable automatic early stopping
)

tuner.fit({'training': f's3://{bucket}/train-data/'})

Quiz 1

Quiz 2

Quiz 3

Quiz 4

Quiz 5