Complete Workflow Example

This example demonstrates a complete end-to-end workflow using OptiScope, from loading data through analysis to visualization and saving results.

Overview

This workflow covers:

1. Loading optimization results
2. Identifying the Pareto front
3. Filtering to a representative subset
4. Ranking solutions with TOPSIS
5. Detecting knee points
6. Creating result sets
7. Comprehensive visualization
8. Saving results

Complete Code

from optiscope.io import load_results, save_results
from optiscope.analysis import (
    identify_pareto_front,
    adaptive_smart_pareto_filter,
    topsis,
    detect_knee_points,
    compare_weight_scenarios
)
from optiscope.plotting import (
    plot_parallel_coordinates,
    plot_pareto_front_2d,
    plot_scatter_matrix,
    plot_correlation_heatmap
)
from optiscope.plotting.time_series import plot_optimization_history
from optiscope.core.result_set import ResultSet
import numpy as np

# ============================================================================
# Step 1: Load Data
# ============================================================================
print("=" * 80)
print("STEP 1: Loading optimization results")
print("=" * 80)

result = load_results("optimization_results.csv")
print(f"✓ Loaded {len(result.objectives)} solutions")
print(f"  - Objectives: {result.objective_names}")
print(f"  - Design variables: {result.design_var_names}")
print(f"  - Optimization directions: {result.optimization_directions}")

# ============================================================================
# Step 2: Identify Pareto Front
# ============================================================================
print("\n" + "=" * 80)
print("STEP 2: Identifying Pareto front")
print("=" * 80)

pareto_mask = identify_pareto_front(
    objectives=result.objectives.values,
    directions=result.optimization_directions
)
pareto_indices = np.where(pareto_mask)[0]
print(f"✓ Found {len(pareto_indices)} Pareto-optimal solutions")
print(f"  ({len(pareto_indices)/len(result.objectives)*100:.1f}% of total)")

# Create Pareto front result set
result.add_set(ResultSet(
    name="pareto_front",
    indices=pareto_indices.tolist(),
    created_by="analysis_workflow",
    description="All Pareto-optimal solutions"
))

# ============================================================================
# Step 3: Apply Smart Pareto Filter
# ============================================================================
print("\n" + "=" * 80)
print("STEP 3: Applying Smart Pareto Filter")
print("=" * 80)

# Only filter if Pareto front is large
if len(pareto_indices) > 50:
    filtered_indices = adaptive_smart_pareto_filter(
        objectives=result.objectives.values[pareto_indices],
        target_reduction=0.3,  # Reduce to 30% of Pareto front
        min_points=10,
        max_points=50,
        normalize=True
    )
    final_indices = pareto_indices[filtered_indices]
    print(f"✓ Reduced from {len(pareto_indices)} to {len(final_indices)} solutions")
    print(f"  (Reduction: {(1 - len(final_indices)/len(pareto_indices))*100:.1f}%)")

    # Create filtered Pareto result set
    result.add_set(ResultSet(
        name="filtered_pareto",
        indices=final_indices.tolist(),
        created_by="smart_pareto_filter",
        description=f"Representative subset of Pareto front ({len(final_indices)} points)"
    ))
else:
    print(f"✓ Pareto front has only {len(pareto_indices)} solutions - no filtering needed")
    final_indices = pareto_indices

# ============================================================================
# Step 4: Rank with TOPSIS
# ============================================================================
print("\n" + "=" * 80)
print("STEP 4: Ranking solutions with TOPSIS")
print("=" * 80)

# Define weight scenarios for different stakeholders
weight_scenarios = {
    "Balanced": np.array([1/3, 1/3, 1/3]),
    "Cost-focused": np.array([0.6, 0.2, 0.2]),
    "Performance-focused": np.array([0.2, 0.6, 0.2]),
    "Quality-focused": np.array([0.2, 0.2, 0.6])
}

# Compare scenarios
print("Comparing weight scenarios:")
for scenario_name, weights in weight_scenarios.items():
    scores = topsis(
        objectives=result.objectives.values[final_indices],
        weights=weights.tolist(),
        directions=result.optimization_directions
    )
    best_in_filtered = np.argmax(scores)
    best_idx = final_indices[best_in_filtered]
    print(f"  {scenario_name:20s}: Best = index {best_idx:4d}, score = {scores[best_in_filtered]:.3f}")

# Use balanced weights for final ranking
balanced_weights = weight_scenarios["Balanced"]
scores = topsis(
    objectives=result.objectives.values[final_indices],
    weights=balanced_weights.tolist(),
    directions=result.optimization_directions
)

# Get top 5 solutions
top_5_in_filtered = np.argsort(scores)[-5:][::-1]
top_5_indices = final_indices[top_5_in_filtered]

print(f"\n✓ Top 5 solutions (balanced weights):")
for i, idx in enumerate(top_5_indices):
    print(f"  {i+1}. Index {idx:4d}: TOPSIS score = {scores[top_5_in_filtered[i]]:.3f}")
    print(f"     Objectives: {result.objectives.iloc[idx]}")

# Create result sets for top solutions
result.add_set(ResultSet(
    name="top_5_topsis",
    indices=top_5_indices.tolist(),
    created_by="topsis",
    description="Top 5 solutions by TOPSIS (balanced weights)"
))

result.add_set(ResultSet(
    name="best_solution",
    indices=[top_5_indices[0]],
    created_by="topsis",
    description=f"Best solution (TOPSIS score: {scores[top_5_in_filtered[0]]:.3f})"
))

# ============================================================================
# Step 5: Detect Knee Point
# ============================================================================
print("\n" + "=" * 80)
print("STEP 5: Detecting knee point")
print("=" * 80)

# Only detect knee for 2-3 objective problems
if len(result.objective_names) <= 3:
    knee_indices = detect_knee_points(
        objectives=result.objectives.values[final_indices],
        method="angle",
        n_knees=1,
        normalize=True
    )
    knee_idx = final_indices[knee_indices[0]]

    print(f"✓ Knee point detected at index {knee_idx}")
    print(f"  Objectives: {result.objectives.iloc[knee_idx]}")

    # Create knee point result set
    result.add_set(ResultSet(
        name="knee_point",
        indices=[knee_idx],
        created_by="knee_detection",
        description="Best compromise solution (knee point)"
    ))
else:
    print(f"✗ Knee detection skipped (works best for 2-3 objectives, have {len(result.objective_names)})")

# ============================================================================
# Step 6: Summary of Result Sets
# ============================================================================
print("\n" + "=" * 80)
print("STEP 6: Summary of created result sets")
print("=" * 80)

print(f"Created {len(result.sets)} result sets:")
for name, rs in result.sets.items():
    print(f"  • {name:20s}: {len(rs.indices):4d} solutions - {rs.description}")

# ============================================================================
# Step 7: Visualization
# ============================================================================
print("\n" + "=" * 80)
print("STEP 7: Creating visualizations")
print("=" * 80)

# 7.1 Parallel Coordinates - Overview
print("  Creating parallel coordinates plot...")
fig_parallel = plot_parallel_coordinates(
    result,
    result_sets=["pareto_front", "filtered_pareto", "top_5_topsis", "best_solution"],
    color_by="Set",
    title="Optimization Analysis - Parallel Coordinates"
)
fig_parallel.write_html("output/1_parallel_coordinates.html")
print("    ✓ Saved to output/1_parallel_coordinates.html")

# 7.2 Pareto Front Visualization (if 2-3 objectives)
if len(result.objective_names) == 2:
    print("  Creating 2D Pareto front plot...")
    fig_pareto = plot_pareto_front_2d(
        result,
        obj_x=0,
        obj_y=1,
        result_sets=["pareto_front", "filtered_pareto", "best_solution"],
        color_by="Set",
        show_dominated=True,
        title="Pareto Front Analysis"
    )
    fig_pareto.write_html("output/2_pareto_front.html")
    print("    ✓ Saved to output/2_pareto_front.html")
elif len(result.objective_names) == 3:
    print("  Creating 3D Pareto front plot...")
    from optiscope.plotting import plot_pareto_front_3d
    fig_pareto = plot_pareto_front_3d(
        result,
        obj_x=0,
        obj_y=1,
        obj_z=2,
        result_sets=["pareto_front", "filtered_pareto", "best_solution"],
        color_by="Set",
        title="Pareto Front Analysis - 3D"
    )
    fig_pareto.write_html("output/2_pareto_front_3d.html")
    print("    ✓ Saved to output/2_pareto_front_3d.html")

# 7.3 Scatter Matrix - Objective Space
print("  Creating scatter matrix...")
objective_names = [f"objective_{i}" for i in range(len(result.objective_names))]
fig_splom = plot_scatter_matrix(
    result,
    variables=objective_names[:min(5, len(objective_names))],  # Limit to 5 for readability
    result_sets=["filtered_pareto", "top_5_topsis", "best_solution"],
    color_by="Set",
    show_distributions=True,
    title="Objective Space - Scatter Matrix"
)
fig_splom.write_html("output/3_scatter_matrix.html")
print("    ✓ Saved to output/3_scatter_matrix.html")

# 7.4 Correlation Heatmap
print("  Creating correlation heatmap...")
fig_corr = plot_correlation_heatmap(
    result,
    include_objectives=True,
    include_design_vars=True,
    include_constraints=False,
    method="pearson",
    title="Variable Correlations"
)
fig_corr.write_html("output/4_correlations.html")
print("    ✓ Saved to output/4_correlations.html")

# 7.5 Optimization History (if available)
if hasattr(result, 'iteration') or result.objectives.shape[0] > 1:
    print("  Creating optimization history plots...")
    figs = plot_optimization_history(
        result,
        show_markers=False,
        show_feasibility=True
    )
    for i, (category, fig) in enumerate(figs.items()):
        fig.write_html(f"output/5_{category}_history.html")
        print(f"    ✓ Saved to output/5_{category}_history.html")

print("\n✓ All visualizations created successfully!")

# ============================================================================
# Step 8: Save Results
# ============================================================================
print("\n" + "=" * 80)
print("STEP 8: Saving results")
print("=" * 80)

# Save result with all result sets
save_results(result, "output/analyzed_results.csv")
print("✓ Saved analyzed results to output/analyzed_results.csv")

# Save a summary report
with open("output/analysis_summary.txt", "w") as f:
    f.write("OPTIMIZATION ANALYSIS SUMMARY\n")
    f.write("=" * 80 + "\n\n")

    f.write(f"Total solutions: {len(result.objectives)}\n")
    f.write(f"Objectives: {result.objective_names}\n")
    f.write(f"Design variables: {result.design_var_names}\n\n")

    f.write("RESULT SETS:\n")
    f.write("-" * 80 + "\n")
    for name, rs in result.sets.items():
        f.write(f"{name}:\n")
        f.write(f"  Solutions: {len(rs.indices)}\n")
        f.write(f"  Created by: {rs.created_by}\n")
        f.write(f"  Description: {rs.description}\n\n")

    f.write("BEST SOLUTION (TOPSIS - Balanced Weights):\n")
    f.write("-" * 80 + "\n")
    best_idx = top_5_indices[0]
    f.write(f"Index: {best_idx}\n")
    f.write(f"TOPSIS Score: {scores[top_5_in_filtered[0]]:.3f}\n")
    f.write(f"Objectives: {result.objectives.iloc[best_idx]}\n")
    f.write(f"Design Variables: {result.design_vars.iloc[best_idx]}\n")

print("✓ Saved analysis summary to output/analysis_summary.txt")

# ============================================================================
# Complete!
# ============================================================================
print("\n" + "=" * 80)
print("ANALYSIS COMPLETE!")
print("=" * 80)
print("\nGenerated files:")
print("  • output/analyzed_results.csv - Results with all result sets")
print("  • output/analysis_summary.txt - Text summary of analysis")
print("  • output/1_parallel_coordinates.html - Interactive parallel coordinates")
print("  • output/2_pareto_front.html - Pareto front visualization")
print("  • output/3_scatter_matrix.html - Objective space SPLOM")
print("  • output/4_correlations.html - Correlation heatmap")
print("  • output/5_*_history.html - Optimization history plots")
print("\nRecommended solution: Index", top_5_indices[0])
print("=" * 80)

Running the Workflow

Prerequisites

# Install OptiScope with all dependencies
pip install optiscope[all]

# Or install specific dependencies
pip install optiscope plotly-resampler kaleido

Prepare Your Data

Ensure your optimization results are in a supported format (CSV, JSON, HDF5, or Parquet):

# If you need to convert your data, see the loading_data.md guide

Create Output Directory

mkdir -p output

Run the Workflow

python complete_workflow.py

Customizing the Workflow

Adjust TOPSIS Weights

Modify the weight scenarios based on your priorities:

weight_scenarios = {
    "Your_Scenario": np.array([0.5, 0.3, 0.2]),  # Adjust weights
    # Add more scenarios as needed
}

Change Smart Pareto Filter Settings

Adjust the target reduction and constraints:

filtered_indices = adaptive_smart_pareto_filter(
    objectives=result.objectives.values[pareto_indices],
    target_reduction=0.2,  # More aggressive filtering (20% of Pareto front)
    min_points=15,  # Keep at least 15 points
    max_points=30,  # Keep at most 30 points
    normalize=True
)

Select Different Visualization Variables

Choose which variables to include in plots:

# For scatter matrix, select specific variables
selected_vars = ["design_var_0", "design_var_3", "objective_0", "objective_1"]
fig_splom = plot_scatter_matrix(
    result,
    variables=selected_vars,
    # ... other parameters
)

Skip Steps

Comment out steps you don't need:

# # Skip knee detection
# if len(result.objective_names) <= 3:
#     knee_indices = detect_knee_points(...)

Output Interpretation

Result Sets

The workflow creates several result sets:

• pareto_front: All non-dominated solutions
• filtered_pareto: Representative subset of Pareto front
• top_5_topsis: Top 5 solutions by TOPSIS ranking
• best_solution: Single best solution
• knee_point: Best compromise solution (if applicable)

Visualizations

1. Parallel Coordinates: Overview of all result sets across all variables
2. Pareto Front: Trade-offs between objectives
3. Scatter Matrix: Pairwise relationships in objective space
4. Correlations: Variable dependencies
5. History Plots: Convergence and evolution over iterations

Analysis Summary

The analysis_summary.txt file contains:

• Dataset statistics
• Result set descriptions
• Best solution details

Next Steps

After running this workflow:

1. Review visualizations in the output/ directory
2. Examine the best solution and understand why it was selected
3. Compare weight scenarios to see how preferences affect rankings
4. Iterate if needed by adjusting weights or filtering parameters
5. Export specific solutions for further evaluation or implementation

Troubleshooting

Memory Issues

For very large datasets:

# Filter early to reduce memory usage
if len(result.objectives) > 100000:
    # Keep only top 20% for each objective
    masks = []
    for i in range(result.objectives.shape[1]):
        threshold = np.percentile(result.objectives[:, i], 20)
        masks.append(result.objectives[:, i] <= threshold)
    combined_mask = np.any(masks, axis=0)
    result = result[combined_mask]  # Filter result

Missing Dependencies

If you get import errors:

# Install missing packages
pip install plotly-resampler  # For large time series
pip install kaleido  # For saving static images
pip install h5py  # For HDF5 support
pip install pyarrow  # For Parquet support

No Pareto Front Found

If no Pareto front is found, check:

# Verify optimization directions are correct
print(result.metadata.optimization_directions)

# Check for duplicate solutions
unique_objectives = np.unique(result.objectives.values, axis=0)
print(f"Unique solutions: {len(unique_objectives)} / {len(result.objectives)}")