From dc1842ef5a0e52eb75bea5006caf77761ed6cda3 Mon Sep 17 00:00:00 2001 From: Tobias Brunner Date: Fri, 20 Jun 2025 15:53:57 +0200 Subject: [PATCH] improve performane on full pricelist view --- hub/services/views/pricelist.py | 379 ++++++++++---------------------- 1 file changed, 111 insertions(+), 268 deletions(-) diff --git a/hub/services/views/pricelist.py b/hub/services/views/pricelist.py index add5b22..b39c975 100644 --- a/hub/services/views/pricelist.py +++ b/hub/services/views/pricelist.py @@ -124,7 +124,7 @@ def get_internal_cloud_provider_comparisons( @staff_member_required def pricelist(request): - """Generate comprehensive price list grouped by compute plan groups and service levels""" + """Generate comprehensive price list grouped by compute plan groups and service levels (optimized)""" # Get filter parameters from request show_discount_details = request.GET.get("discount_details", "").lower() == "true" show_addon_details = request.GET.get("addon_details", "").lower() == "true" @@ -134,45 +134,38 @@ def pricelist(request): filter_compute_plan_group = request.GET.get("compute_plan_group", "") filter_service_level = request.GET.get("service_level", "") - # Fetch all active compute plans with related data - compute_plans = ( - ComputePlan.objects.filter(active=True) - .select_related("cloud_provider", "group") - .prefetch_related("prices") - .order_by("group__order", "group__name", "cloud_provider__name") - ) - - # Apply compute plan filters + # Fetch all active compute plans with related data (move as much sorting/filtering to DB as possible) + compute_plans_qs = ComputePlan.objects.filter(active=True) if filter_cloud_provider: - compute_plans = compute_plans.filter(cloud_provider__name=filter_cloud_provider) + compute_plans_qs = compute_plans_qs.filter(cloud_provider__name=filter_cloud_provider) if filter_compute_plan_group: if filter_compute_plan_group == "No Group": - compute_plans = compute_plans.filter(group__isnull=True) + compute_plans_qs = compute_plans_qs.filter(group__isnull=True) else: - compute_plans = compute_plans.filter(group__name=filter_compute_plan_group) - - # Apply natural sorting for compute plan names - compute_plans = sorted( - compute_plans, - key=lambda x: ( - x.group.order if x.group else 999, # No group plans at the end - x.group.name if x.group else "ZZZ", - x.cloud_provider.name, - natural_sort_key(x.name), - ), + compute_plans_qs = compute_plans_qs.filter(group__name=filter_compute_plan_group) + compute_plans = list( + compute_plans_qs + .select_related("cloud_provider", "group") + .prefetch_related("prices") + .order_by("group__order", "group__name", "cloud_provider__name", "name") ) - # Fetch all appcat price configurations - appcat_prices = ( + # Fetch all appcat price configurations (prefetch addons) + appcat_prices_qs = ( VSHNAppCatPrice.objects.all() .select_related("service", "discount_model") - .prefetch_related("base_fees", "unit_rates", "discount_model__tiers") + .prefetch_related("base_fees", "unit_rates", "discount_model__tiers", "addons") .order_by("service__name") ) - - # Apply service filter if filter_service: - appcat_prices = appcat_prices.filter(service__name=filter_service) + appcat_prices_qs = appcat_prices_qs.filter(service__name=filter_service) + appcat_prices = list(appcat_prices_qs) + + # Prefetch all storage plans for all cloud providers and build a lookup + all_storage_plans = StoragePlan.objects.all().prefetch_related("prices") + storage_plans_by_provider = defaultdict(list) + for sp in all_storage_plans: + storage_plans_by_provider[sp.cloud_provider_id].append(sp) pricing_data_by_group_and_service_level = defaultdict(lambda: defaultdict(list)) processed_combinations = set() @@ -180,7 +173,6 @@ def pricelist(request): # Generate pricing combinations for each compute plan and service for plan in compute_plans: plan_currencies = set(plan.prices.values_list("currency", flat=True)) - for appcat_price in appcat_prices: # Determine units based on variable unit type if appcat_price.variable_unit == VSHNAppCatPrice.VariableUnit.RAM: @@ -189,39 +181,22 @@ def pricelist(request): units = int(plan.vcpus) else: continue - - base_fee_currencies = set( - appcat_price.base_fees.values_list("currency", flat=True) - ) - - service_levels = appcat_price.unit_rates.values_list( - "service_level", flat=True - ).distinct() - + base_fee_currencies = set(appcat_price.base_fees.values_list("currency", flat=True)) + service_levels = appcat_price.unit_rates.values_list("service_level", flat=True).distinct() # Apply service level filter if filter_service_level: service_levels = [ - sl - for sl in service_levels - if dict(VSHNAppCatPrice.ServiceLevel.choices)[sl] - == filter_service_level + sl for sl in service_levels + if dict(VSHNAppCatPrice.ServiceLevel.choices)[sl] == filter_service_level ] - for service_level in service_levels: unit_rate_currencies = set( - appcat_price.unit_rates.filter( - service_level=service_level - ).values_list("currency", flat=True) + appcat_price.unit_rates.filter(service_level=service_level).values_list("currency", flat=True) ) - # Find currencies that exist across all pricing components - matching_currencies = plan_currencies.intersection( - base_fee_currencies - ).intersection(unit_rate_currencies) - + matching_currencies = plan_currencies.intersection(base_fee_currencies).intersection(unit_rate_currencies) if not matching_currencies: continue - for currency in matching_currencies: combination_key = ( plan.cloud_provider.name, @@ -230,63 +205,35 @@ def pricelist(request): service_level, currency, ) - - # Skip if combination already processed if combination_key in processed_combinations: continue - processed_combinations.add(combination_key) - # Get pricing components compute_plan_price = plan.get_price(currency) base_fee = appcat_price.get_base_fee(currency, service_level) unit_rate = appcat_price.get_unit_rate(currency, service_level) - - # Skip if any pricing component is missing - if any( - price is None - for price in [compute_plan_price, base_fee, unit_rate] - ): + if any(price is None for price in [compute_plan_price, base_fee, unit_rate]): continue - # Calculate replica enforcement based on service level if service_level == VSHNAppCatPrice.ServiceLevel.GUARANTEED: replica_enforce = appcat_price.ha_replica_min else: replica_enforce = 1 - total_units = units * replica_enforce standard_sla_price = base_fee + (total_units * unit_rate) - # Apply discount if available discount_breakdown = None - if ( - appcat_price.discount_model - and appcat_price.discount_model.active - ): - discounted_price = ( - appcat_price.discount_model.calculate_discount( - unit_rate, total_units - ) - ) + if appcat_price.discount_model and appcat_price.discount_model.active: + discounted_price = appcat_price.discount_model.calculate_discount(unit_rate, total_units) sla_price = base_fee + discounted_price discount_savings = standard_sla_price - sla_price - discount_percentage = ( - (discount_savings / standard_sla_price) * 100 - if standard_sla_price > 0 - else 0 - ) - discount_breakdown = ( - appcat_price.discount_model.get_discount_breakdown( - unit_rate, total_units - ) - ) + discount_percentage = (discount_savings / standard_sla_price) * 100 if standard_sla_price > 0 else 0 + discount_breakdown = appcat_price.discount_model.get_discount_breakdown(unit_rate, total_units) else: sla_price = standard_sla_price discounted_price = total_units * unit_rate discount_savings = 0 discount_percentage = 0 - # Calculate final price using the model method to ensure consistency price_calculation = appcat_price.calculate_final_price( currency_code=currency, @@ -294,60 +241,22 @@ def pricelist(request): number_of_units=total_units, addon_ids=None, # This will include only mandatory addons ) - if price_calculation is None: continue - # Calculate base service price (without addons) for display purposes base_sla_price = base_fee + (total_units * unit_rate) - - # Apply discount if available - discount_breakdown = None - if ( - appcat_price.discount_model - and appcat_price.discount_model.active - ): - discounted_price = ( - appcat_price.discount_model.calculate_discount( - unit_rate, total_units - ) - ) - sla_price = base_fee + discounted_price - discount_savings = base_sla_price - sla_price - discount_percentage = ( - (discount_savings / base_sla_price) * 100 - if base_sla_price > 0 - else 0 - ) - discount_breakdown = ( - appcat_price.discount_model.get_discount_breakdown( - unit_rate, total_units - ) - ) - else: - sla_price = base_sla_price - discounted_price = total_units * unit_rate - discount_savings = 0 - discount_percentage = 0 - - # Extract addon information from the calculation + # Extract addon information from the calculation (use prefetched addons) mandatory_addons = [] optional_addons = [] - - # Get all addons to separate mandatory from optional - all_addons = appcat_price.addons.filter(active=True) + all_addons = [a for a in appcat_price.addons.all() if a.active] for addon in all_addons: addon_price = None - if addon.addon_type == "BF": # Base Fee addon_price = addon.get_price(currency, service_level) elif addon.addon_type == "UR": # Unit Rate - addon_price_per_unit = addon.get_price( - currency, service_level - ) + addon_price_per_unit = addon.get_price(currency, service_level) if addon_price_per_unit: addon_price = addon_price_per_unit * total_units - addon_info = { "id": addon.id, "name": addon.name, @@ -356,165 +265,102 @@ def pricelist(request): "addon_type": addon.get_addon_type_display(), "price": addon_price, } - if addon.mandatory: mandatory_addons.append(addon_info) else: optional_addons.append(addon_info) - - # Use the calculated total price which includes mandatory addons service_price_with_addons = price_calculation["total_price"] final_price = compute_plan_price + service_price_with_addons - service_level_display = dict(VSHNAppCatPrice.ServiceLevel.choices)[ - service_level - ] - - # Get external price comparisons if enabled + service_level_display = dict(VSHNAppCatPrice.ServiceLevel.choices).get(service_level, service_level) + # Get external/internal price comparisons if enabled (unchanged, but could be optimized further) external_comparisons = [] internal_comparisons = [] if show_price_comparison: - # Get external price comparisons - external_prices = get_external_price_comparisons( - plan, appcat_price, currency, service_level - ) + external_prices = get_external_price_comparisons(plan, appcat_price, currency, service_level) for ext_price in external_prices: - # Calculate price difference using external price currency difference = ext_price.amount - final_price - ratio = ( - ext_price.amount / final_price if final_price > 0 else 0 - ) - - external_comparisons.append( - { - "plan_name": ext_price.plan_name, - "provider": ext_price.cloud_provider.name, - "description": ext_price.description, - "amount": ext_price.amount, - "currency": ext_price.currency, # Use external price currency - "vcpus": ext_price.vcpus, - "ram": ext_price.ram, - "storage": ext_price.storage, - "replicas": ext_price.replicas, - "difference": difference, - "ratio": ratio, - "source": ext_price.source, - "date_retrieved": ext_price.date_retrieved, - "is_internal": False, - } - ) - - # Get internal cloud provider comparisons - internal_price_comparisons = ( - get_internal_cloud_provider_comparisons( - plan, appcat_price, currency, service_level - ) - ) + ratio = ext_price.amount / final_price if final_price > 0 else 0 + external_comparisons.append({ + "plan_name": ext_price.plan_name, + "provider": ext_price.cloud_provider.name, + "description": ext_price.description, + "amount": ext_price.amount, + "currency": ext_price.currency, + "vcpus": ext_price.vcpus, + "ram": ext_price.ram, + "storage": ext_price.storage, + "replicas": ext_price.replicas, + "difference": difference, + "ratio": ratio, + "source": ext_price.source, + "date_retrieved": ext_price.date_retrieved, + "is_internal": False, + }) + internal_price_comparisons = get_internal_cloud_provider_comparisons(plan, appcat_price, currency, service_level) for int_price in internal_price_comparisons: - # Calculate price difference difference = int_price["final_price"] - final_price - ratio = ( - int_price["final_price"] / final_price - if final_price > 0 - else 0 - ) - - internal_comparisons.append( - { - "plan_name": int_price["plan_name"], - "provider": int_price["provider"], - "description": f"Same specs with {int_price['provider']}", - "amount": int_price["final_price"], - "currency": int_price["currency"], - "vcpus": int_price["vcpus"], - "ram": int_price["ram"], - "group_name": int_price["group_name"], - "compute_plan_price": int_price[ - "compute_plan_price" - ], - "service_price": int_price["service_price"], - "difference": difference, - "ratio": ratio, - "is_internal": True, - } - ) - + ratio = int_price["final_price"] / final_price if final_price > 0 else 0 + internal_comparisons.append({ + "plan_name": int_price["plan_name"], + "provider": int_price["provider"], + "description": f"Same specs with {int_price['provider']}", + "amount": int_price["final_price"], + "currency": int_price["currency"], + "vcpus": int_price["vcpus"], + "ram": int_price["ram"], + "group_name": int_price["group_name"], + "compute_plan_price": int_price["compute_plan_price"], + "service_price": int_price["service_price"], + "difference": difference, + "ratio": ratio, + "is_internal": True, + }) group_name = plan.group.name if plan.group else "No Group" - - # Get storage plans for this cloud provider - storage_plans = StoragePlan.objects.filter( - cloud_provider=plan.cloud_provider - ).prefetch_related("prices") - - # Add pricing data to the grouped structure - pricing_data_by_group_and_service_level[group_name][ - service_level_display - ].append( - { - "cloud_provider": plan.cloud_provider.name, - "service": appcat_price.service.name, - "compute_plan": plan.name, - "compute_plan_group": group_name, - "compute_plan_group_description": ( - plan.group.description if plan.group else "" - ), - "compute_plan_group_node_label": ( - plan.group.node_label if plan.group else "" - ), - "storage_plans": storage_plans, - "vcpus": plan.vcpus, - "ram": plan.ram, - "cpu_mem_ratio": plan.cpu_mem_ratio, - "term": plan.get_term_display(), - "currency": currency, - "compute_plan_price": compute_plan_price, - "variable_unit": appcat_price.get_variable_unit_display(), - "units": units, - "replica_enforce": replica_enforce, - "total_units": total_units, - "service_level": service_level_display, - "sla_base": base_fee, - "sla_per_unit": unit_rate, - "sla_price": service_price_with_addons, - "standard_sla_price": base_sla_price, - "discounted_sla_price": ( - base_fee + discounted_price - if appcat_price.discount_model - and appcat_price.discount_model.active - else None - ), - "discount_savings": discount_savings, - "discount_percentage": discount_percentage, - "discount_breakdown": discount_breakdown, - "final_price": final_price, - "discount_model": ( - appcat_price.discount_model.name - if appcat_price.discount_model - else None - ), - "has_discount": bool( - appcat_price.discount_model - and appcat_price.discount_model.active - ), - "external_comparisons": external_comparisons, - "internal_comparisons": internal_comparisons, - "mandatory_addons": mandatory_addons, - "optional_addons": optional_addons, - } - ) - + # Use prefetched storage plans + storage_plans = storage_plans_by_provider.get(plan.cloud_provider_id, []) + pricing_data_by_group_and_service_level[group_name][service_level_display].append({ + "cloud_provider": plan.cloud_provider.name, + "service": appcat_price.service.name, + "compute_plan": plan.name, + "compute_plan_group": group_name, + "compute_plan_group_description": (plan.group.description if plan.group else ""), + "compute_plan_group_node_label": (plan.group.node_label if plan.group else ""), + "storage_plans": storage_plans, + "vcpus": plan.vcpus, + "ram": plan.ram, + "cpu_mem_ratio": plan.cpu_mem_ratio, + "term": plan.get_term_display(), + "currency": currency, + "compute_plan_price": compute_plan_price, + "variable_unit": appcat_price.get_variable_unit_display(), + "units": units, + "replica_enforce": replica_enforce, + "total_units": total_units, + "service_level": service_level_display, + "sla_base": base_fee, + "sla_per_unit": unit_rate, + "sla_price": service_price_with_addons, + "standard_sla_price": base_sla_price, + "discounted_sla_price": (base_fee + discounted_price if appcat_price.discount_model and appcat_price.discount_model.active else None), + "discount_savings": discount_savings, + "discount_percentage": discount_percentage, + "discount_breakdown": discount_breakdown, + "final_price": final_price, + "discount_model": (appcat_price.discount_model.name if appcat_price.discount_model else None), + "has_discount": bool(appcat_price.discount_model and appcat_price.discount_model.active), + "external_comparisons": external_comparisons, + "internal_comparisons": internal_comparisons, + "mandatory_addons": mandatory_addons, + "optional_addons": optional_addons, + }) # Order groups correctly, placing "No Group" last ordered_groups_intermediate = {} all_group_names = list(pricing_data_by_group_and_service_level.keys()) - if "No Group" in all_group_names: all_group_names.remove("No Group") all_group_names.append("No Group") - for group_name_key in all_group_names: - ordered_groups_intermediate[group_name_key] = ( - pricing_data_by_group_and_service_level[group_name_key] - ) - + ordered_groups_intermediate[group_name_key] = pricing_data_by_group_and_service_level[group_name_key] # Convert defaultdicts to regular dicts for the template final_context_data = {} for group_key, service_levels_dict in ordered_groups_intermediate.items(): @@ -522,7 +368,6 @@ def pricelist(request): sl_key: list(plans_list) for sl_key, plans_list in service_levels_dict.items() } - # Get filter options for dropdowns all_cloud_providers = ( ComputePlan.objects.filter(active=True) @@ -543,13 +388,11 @@ def pricelist(request): ) all_compute_plan_groups.append("No Group") # Add option for plans without groups all_service_levels = [choice[1] for choice in VSHNAppCatPrice.ServiceLevel.choices] - # If no filter is specified, select the first available provider/service by default if not filter_cloud_provider and all_cloud_providers: filter_cloud_provider = all_cloud_providers[0] if not filter_service and all_services: filter_service = all_services[0] - context = { "pricing_data_by_group_and_service_level": final_context_data, "show_discount_details": show_discount_details,