This documentation is built from the development branch, master. It includes the changes listed under Unreleased, which the latest release, CityLearn 3.6.0, does not have. To read the documentation of the version you installed (pip show citylearn), open the docs folder at its tag on GitHub, for example v3.6.0.

Source code for citylearn.reward_function

import importlib
from typing import Any, Iterable, List, Mapping, Optional, Set, Tuple, Union
import numpy as np
from citylearn.building import Building
from citylearn.data import ZERO_DIVISION_PLACEHOLDER
from citylearn.internal.scalars import numpy1_scalar
import logging

LOGGER = logging.getLogger(__name__)
[docs] class RewardFunction: r"""Base and default reward function class. The default reward is the electricity consumption from the grid at the current time step returned as a negative value. Parameters ---------- env_metadata: Mapping[str, Any]: General static information about the environment. **kwargs : dict Other keyword arguments for custom reward calculation. """ def __init__(self, env_metadata: Mapping[str, Any], exponent: float = None, **kwargs): penalty_coefficient = kwargs.pop('charging_constraint_penalty_coefficient', None) self.env_metadata = env_metadata self.exponent = exponent self.charging_constraint_penalty_coefficient = penalty_coefficient @property def env_metadata(self) -> Mapping[str, Any]: return self._env_metadata @env_metadata.setter def env_metadata(self, env_metadata: Mapping[str, Any]): self._env_metadata = env_metadata @property def central_agent(self) -> bool: """Expect 1 central agent to control all buildings.""" return self.env_metadata['central_agent'] @property def exponent(self) -> float: return self.__exponent @exponent.setter def exponent(self, exponent: float): self.__exponent = 1.0 if exponent is None else exponent @property def charging_constraint_penalty_coefficient(self) -> float: return getattr(self, '_charging_constraint_penalty_coefficient', 1.0) @charging_constraint_penalty_coefficient.setter def charging_constraint_penalty_coefficient(self, coefficient: float): if coefficient is None: self._charging_constraint_penalty_coefficient = 1.0 else: self._charging_constraint_penalty_coefficient = float(coefficient)
[docs] def reset(self): """Use to reset variables at the start of an episode.""" pass
@staticmethod def _parse_version(version: Optional[int], supported: Tuple[int, ...], owner: str) -> int: version = supported[0] if version is None else version if isinstance(version, bool) or not isinstance(version, (int, np.integer)) or int(version) not in supported: raise ValueError(f'{owner} version must be one of {list(supported)}, got {version!r}.') return int(version)
[docs] def get_required_observation_names(self) -> Optional[Iterable[str]]: """Declare inputs of the built-in calculation; custom rewards remain unrestricted.""" if type(self) is RewardFunction and 'calculate' not in self.__dict__: return ('net_electricity_consumption',) return None
[docs] def calculate(self, observations: List[Mapping[str, Union[int, float]]]) -> List[float]: r"""Calculates reward. Parameters ---------- observations: List[Mapping[str, Union[int, float]]] List of all building observations at current :py:attr:`citylearn.citylearn.CityLearnEnv. time_step` that are got from calling :py:meth:`citylearn.building.Building.observations`. Returns ------- reward: List[float] Reward for transition to current timestep. """ net_electricity_consumption = [o['net_electricity_consumption'] for o in observations] reward_list = [-(max(o, 0)**numpy1_scalar(self.exponent)) for o in net_electricity_consumption] if self.central_agent: reward = [sum(reward_list, numpy1_scalar(0))] else: reward = reward_list return reward
[docs] class MultiBuildingRewardFunction(RewardFunction): """Applies one reward function per building. Used when the schema's ``reward_function.type`` maps building names to reward functions. Each function receives one building's observations and returns that building's reward; the values are returned in building order, or summed as ``[total]`` with a central agent. Each function's ``env_metadata`` is the environment metadata with ``buildings`` reduced to its own building, so functions that read building metadata, such as :py:class:`SolarPenaltyReward`, see that building's assets. Functions are paired with the observations by building name, following ``env_metadata['buildings']``, which the environment refreshes when members join or leave under a dynamic topology. Without building names in the metadata, functions are paired with the observations by position. Up to CityLearn 3.6.0 every function received the metadata of all buildings and read the first entry, i.e. the first building's assets, and functions were always paired by position. Parameters ---------- env: Mapping[str, Any] General static information about the environment (``None`` until the environment sets it). reward_functions: dict[str, RewardFunction] Reward function of each building, keyed by building name. """ def __init__(self, env, reward_functions: dict[str, RewardFunction]): self.env = env self.reward_functions = reward_functions super().__init__(env) def _paired_reward_functions(self, count: int) -> List[Tuple[str, RewardFunction]]: names = self._building_names() if names is None: names = list(self.reward_functions)[:count] if len(names) != count: raise ValueError(f'Got observations for {count} buildings but metadata for {len(names)} buildings.') pairs = [] for name in names: rf = self.reward_functions.get(name) if rf is None: raise ValueError(f"No reward function for building '{name}'") pairs.append((name, rf)) return pairs def _building_names(self) -> Optional[List[str]]: buildings = self.env_metadata.get('buildings') if isinstance(self.env_metadata, Mapping) else None if not isinstance(buildings, list) or not all(isinstance(b, Mapping) and 'name' in b for b in buildings): return None return [b['name'] for b in buildings]
[docs] def calculate(self, observations: list[dict]) -> list[float]: # Each building's function sees only that building, so it yields one value; collect them flat # and aggregate for a central agent like every other reward function. rewards = [] for obs, (_, rf) in zip(observations, self._paired_reward_functions(len(observations))): rewards.extend(float(value) for value in np.asarray(rf.calculate([obs]), dtype=float).reshape(-1)) return [sum(rewards)] if self.central_agent else rewards
[docs] def reset(self): for rf in self.reward_functions.values(): rf.reset()
[docs] def get_required_observation_names(self) -> Optional[Iterable[str]]: required: Set[str] = set() for rf in self.reward_functions.values(): names = rf.get_required_observation_names() if names is None: return None required.update(names) return required
@property def env_metadata(self): return self._env_metadata @env_metadata.setter def env_metadata(self, env_metadata: Mapping[str, Any]): self._env_metadata = env_metadata buildings = self._building_names() for name, rf in self.reward_functions.items(): if buildings is None: rf.env_metadata = env_metadata else: # Inactive members get no building entry; the environment refreshes the metadata when they join. own = [b for b in env_metadata['buildings'] if b['name'] == name] rf.env_metadata = {**env_metadata, 'buildings': own}
[docs] class MARL(RewardFunction): """MARL reward function class. Parameters ---------- env_metadata: Mapping[str, Any]: General static information about the environment. """ def __init__(self, env_metadata: Mapping[str, Any]): super().__init__(env_metadata)
[docs] def get_required_observation_names(self) -> Optional[Iterable[str]]: return ('net_electricity_consumption',)
@staticmethod def _building_rewards(observations: List[Mapping[str, Union[int, float]]]) -> np.ndarray: """Reward of each building, before aggregation for a central agent.""" net_electricity_consumption = [o['net_electricity_consumption'] for o in observations] district_electricity_consumption = sum(net_electricity_consumption, numpy1_scalar(0)) building_electricity_consumption = np.array(net_electricity_consumption, dtype=float)*-1 return np.sign(building_electricity_consumption)*0.01*building_electricity_consumption**2*np.nanmax([0, district_electricity_consumption])
[docs] def calculate(self, observations: List[Mapping[str, Union[int, float]]]) -> List[float]: reward_list = self._building_rewards(observations) if self.central_agent: reward = [reward_list.sum()] else: reward = reward_list.tolist() return reward
[docs] class IndependentSACReward(RewardFunction): """Recommended for use with the `SAC` controllers. Returned reward assumes that the building-agents act independently of each other, without sharing information through the reward. Parameters ---------- env_metadata: Mapping[str, Any]: General static information about the environment. """ def __init__(self, env_metadata: Mapping[str, Any]): super().__init__(env_metadata)
[docs] def get_required_observation_names(self) -> Optional[Iterable[str]]: return ('net_electricity_consumption',)
[docs] def calculate(self, observations: List[Mapping[str, Union[int, float]]]) -> List[float]: net_electricity_consumption = [o['net_electricity_consumption'] for o in observations] reward_list = [min(v*numpy1_scalar(-1**3), 0) for v in net_electricity_consumption] if self.central_agent: reward = [sum(reward_list, numpy1_scalar(0))] else: reward = reward_list return reward
[docs] class SolarPenaltyReward(RewardFunction): """The reward is designed to minimize electricity consumption and maximize solar generation to charge energy storage systems. The reward is calculated for each building, i and summed to provide the agent with a reward that is representative of all the building or buildings (in centralized case)it controls. It encourages net-zero energy use by penalizing grid load satisfaction when there is energy in the energy storage systems as well as penalizing net export when the energy storage systems are not fully charged through the penalty term. There is neither penalty nor reward when the energy storage systems are fully charged during net export to the grid. Whereas, when the energy storage systems are charged to capacity and there is net import from the grid the penalty is maximized. Parameters ---------- env_metadata: Mapping[str, Any]: General static information about the environment. """ def __init__(self, env_metadata: Mapping[str, Any]): super().__init__(env_metadata)
[docs] def get_required_observation_names(self) -> Optional[Iterable[str]]: return ( 'net_electricity_consumption', 'cooling_storage_soc', 'heating_storage_soc', 'dhw_storage_soc', 'electrical_storage_soc', )
[docs] def calculate(self, observations: List[Mapping[str, Union[int, float]]]) -> List[float]: reward_list = [] for o, m in zip(observations, self.env_metadata['buildings']): e = o['net_electricity_consumption'] cc = m['cooling_storage']['capacity'] hc = m['heating_storage']['capacity'] dc = m['dhw_storage']['capacity'] ec = m['electrical_storage']['capacity'] cs = o.get('cooling_storage_soc', 0.0) hs = o.get('heating_storage_soc', 0.0) ds = o.get('dhw_storage_soc', 0.0) es = o.get('electrical_storage_soc', 0.0) reward = 0.0 reward += -(1.0 + np.sign(e)*cs)*abs(e) if cc > ZERO_DIVISION_PLACEHOLDER else 0.0 reward += -(1.0 + np.sign(e)*hs)*abs(e) if hc > ZERO_DIVISION_PLACEHOLDER else 0.0 reward += -(1.0 + np.sign(e)*ds)*abs(e) if dc > ZERO_DIVISION_PLACEHOLDER else 0.0 reward += -(1.0 + np.sign(e)*es)*abs(e) if ec > ZERO_DIVISION_PLACEHOLDER else 0.0 reward_list.append(reward) if self.central_agent: reward = [sum(reward_list, numpy1_scalar(0))] else: reward = reward_list return reward
[docs] class ComfortReward(RewardFunction): """Reward for occupant thermal comfort satisfaction. The reward is calculated as the negative difference between the setpoint and indoor dry-bulb temperature raised to some exponent if outside the comfort band. If within the comfort band, the reward is the negative difference when in cooling mode and temperature is below the setpoint or when in heating mode and temperature is above the setpoint. The reward is 0 if within the comfort band and above the setpoint in cooling mode or below the setpoint and in heating mode. Parameters ---------- env_metadata: Mapping[str, Any]: General static information about the environment. band: float, default: 2.0 Setpoint comfort band (+/-). If not provided, the comfort band time series defined in the building file, or the default time series value of 2.0 is used. lower_exponent: float, default = 2.0 Penalty exponent for when in cooling mode but temperature is above setpoint upper boundary or heating mode but temperature is below setpoint lower boundary. higher_exponent: float, default = 2.0 Penalty exponent for when in cooling mode but temperature is below setpoint lower boundary or heating mode but temperature is above setpoint upper boundary. version: int, default: 1 Reward definition version. * ``1``: the reward of CityLearn 3.6.0 and earlier. In cooling (``hvac_mode`` 1) and heating (``hvac_mode`` 2) mode, the side of the set point penalised within the comfort band, and the exponent above the upper boundary, follow whether the heating demand exceeds the cooling demand at the time step, not the mode. In heating mode at 21.5 C with a 20 C set point, the reward is 0 while the heater is off and -1.5 while it runs; passive overheating within the band is not penalised. * ``2``: follows the HVAC mode as described above. In heating mode at 21.5 C with a 20 C set point the reward is -1.5 whether the heater runs or not. In off (``hvac_mode`` 0) and automatic (``hvac_mode`` 3) mode, which do not identify a single service, both versions use the demands to choose the exponent outside the band. Version 1 is the default so that existing schemas and trained policies keep their reward; use version 2 for new work. """ VERSIONS: Tuple[int, ...] = (1, 2) # Class default for instances pickled before the version attribute existed (CityLearn 3.6.0 and earlier). version: int = 1 def __init__(self, env_metadata: Mapping[str, Any], band: float = None, lower_exponent: float = None, higher_exponent: float = None, version: int = None): super().__init__(env_metadata) self.band = band self.lower_exponent = lower_exponent self.higher_exponent = higher_exponent self.version = self._parse_version(version, self.VERSIONS, type(self).__name__) @property def band(self) -> float: return self.__band @property def lower_exponent(self) -> float: return self.__lower_exponent @property def higher_exponent(self) -> float: return self.__higher_exponent @band.setter def band(self, band: float): self.__band = band @lower_exponent.setter def lower_exponent(self, lower_exponent: float): self.__lower_exponent = 2.0 if lower_exponent is None else lower_exponent @higher_exponent.setter def higher_exponent(self, higher_exponent: float): self.__higher_exponent = 2.0 if higher_exponent is None else higher_exponent
[docs] def calculate(self, observations: List[Mapping[str, Union[int, float]]]) -> List[float]: reward_list = [] for o in observations: heating_demand = o.get('heating_demand', 0.0) cooling_demand = o.get('cooling_demand', 0.0) heating = heating_demand > cooling_demand hvac_mode = o['hvac_mode'] if self.version >= 2 and hvac_mode in [1, 2]: heating = hvac_mode == 2 indoor_dry_bulb_temperature = o['indoor_dry_bulb_temperature'] if hvac_mode in [1, 2]: set_point = o['indoor_dry_bulb_temperature_cooling_set_point'] if hvac_mode == 1 else o['indoor_dry_bulb_temperature_heating_set_point'] band = self.band if self.band is not None else o['comfort_band'] lower_bound_comfortable_indoor_dry_bulb_temperature = set_point - numpy1_scalar(band) upper_bound_comfortable_indoor_dry_bulb_temperature = set_point + numpy1_scalar(band) delta = abs(indoor_dry_bulb_temperature - set_point) if indoor_dry_bulb_temperature < lower_bound_comfortable_indoor_dry_bulb_temperature: exponent = self.lower_exponent if hvac_mode == 2 else self.higher_exponent reward = -(delta**numpy1_scalar(exponent)) elif lower_bound_comfortable_indoor_dry_bulb_temperature <= indoor_dry_bulb_temperature < set_point: reward = 0.0 if heating else -delta elif set_point <= indoor_dry_bulb_temperature <= upper_bound_comfortable_indoor_dry_bulb_temperature: reward = -delta if heating else 0.0 else: exponent = self.higher_exponent if heating else self.lower_exponent reward = -(delta**numpy1_scalar(exponent)) else: cooling_set_point = o['indoor_dry_bulb_temperature_cooling_set_point'] heating_set_point = o['indoor_dry_bulb_temperature_heating_set_point'] band = self.band if self.band is not None else o['comfort_band'] lower_bound_comfortable_indoor_dry_bulb_temperature = heating_set_point - numpy1_scalar(band) upper_bound_comfortable_indoor_dry_bulb_temperature = cooling_set_point + numpy1_scalar(band) cooling_delta = indoor_dry_bulb_temperature - cooling_set_point heating_delta = indoor_dry_bulb_temperature - heating_set_point if indoor_dry_bulb_temperature < lower_bound_comfortable_indoor_dry_bulb_temperature: exponent = self.higher_exponent if not heating else self.lower_exponent reward = -(abs(heating_delta)**numpy1_scalar(exponent)) elif lower_bound_comfortable_indoor_dry_bulb_temperature <= indoor_dry_bulb_temperature < heating_set_point: reward = -(abs(heating_delta)) elif heating_set_point <= indoor_dry_bulb_temperature <= cooling_set_point: reward = 0.0 elif cooling_set_point < indoor_dry_bulb_temperature < upper_bound_comfortable_indoor_dry_bulb_temperature: reward = -(abs(cooling_delta)) else: exponent = self.higher_exponent if heating else self.lower_exponent reward = -(abs(cooling_delta)**numpy1_scalar(exponent)) reward_list.append(reward) if self.central_agent: reward = [sum(reward_list, numpy1_scalar(0))] else: reward = reward_list return reward
[docs] def get_required_observation_names(self) -> Optional[Iterable[str]]: return ( 'heating_demand', 'cooling_demand', 'hvac_mode', 'indoor_dry_bulb_temperature', 'indoor_dry_bulb_temperature_cooling_set_point', 'indoor_dry_bulb_temperature_heating_set_point', 'comfort_band', )
[docs] class SolarPenaltyAndComfortReward(RewardFunction): """Addition of :py:class:`citylearn.reward_function.SolarPenaltyReward` and :py:class:`citylearn.reward_function.ComfortReward`. Parameters ---------- env_metadata: Mapping[str, Any]: General static information about the environment. band: float, default = 2.0 Setpoint comfort band (+/-). If not provided, the comfort band time series defined in the building file, or the default time series value of 2.0 is used. lower_exponent: float, default = 2.0 Penalty exponent for when in cooling mode but temperature is above setpoint upper boundary or heating mode but temperature is below setpoint lower boundary. higher_exponent: float, default = 3.0 Penalty exponent for when in cooling mode but temperature is below setpoint lower boundary or heating mode but temperature is above setpoint upper boundary. coefficients: Tuple, default = (1.0, 1.0) Coefficents for `citylearn.reward_function.SolarPenaltyReward` and :py:class:`citylearn.reward_function.ComfortReward` values respectively. version: int, default: 1 Version of the :py:class:`citylearn.reward_function.ComfortReward` term; see its `version` parameter. """ def __init__(self, env_metadata: Mapping[str, Any], band: float = None, lower_exponent: float = None, higher_exponent: float = None, coefficients: Tuple = None, version: int = None): comfort = ComfortReward( env_metadata, band=band, lower_exponent=lower_exponent, higher_exponent=higher_exponent, version=version ) self.__functions: List[RewardFunction] = [SolarPenaltyReward(env_metadata), comfort] self.version = comfort.version super().__init__(env_metadata) self.coefficients = coefficients @property def coefficients(self) -> Tuple: return self.__coefficients @RewardFunction.env_metadata.setter def env_metadata(self, env_metadata: Mapping[str, Any]) -> Mapping[str, Any]: RewardFunction.env_metadata.fset(self, env_metadata) for f in self.__functions: f.env_metadata = self.env_metadata @coefficients.setter def coefficients(self, coefficients: Tuple): coefficients = [1.0]*len(self.__functions) if coefficients is None else coefficients assert len(coefficients) == len(self.__functions), f'{type(self).__name__} needs {len(self.__functions)} coefficients.' self.__coefficients = coefficients
[docs] def calculate(self, observations: List[Mapping[str, Union[int, float]]]) -> List[float]: reward = np.array([f.calculate(observations) for f in self.__functions], dtype='float32') reward = reward*np.reshape(self.coefficients, (len(self.coefficients), 1)) reward = reward.sum(axis=0).tolist() return reward
[docs] def get_required_observation_names(self) -> Optional[Iterable[str]]: required: Set[str] = set() for function in self.__functions: names = function.get_required_observation_names() if names is None: return None required.update(names) return required
[docs] class Electric_Vehicles_Reward_Function(MARL): r"""Reward function for electric vehicle charging behavior in V2G settings. The reward combines the :py:class:`MARL` reward of each building (the base reward), terms for the EV chargers of the building, weighted by `weights`, and a penalty of `charging_constraint_penalty_coefficient` per kWh of `charging_constraint_violation_kwh`. The EV terms of a building are scaled by :math:`1/(1 + |r|)`, where :math:`r` is the base reward. Parameters ---------- env_metadata: Mapping[str, Any] General static information about the environment. weights: Mapping[str, float], optional Signed weights of the EV terms: `no_car_charging`, `battery_limits`, `soc_impossible`, `soc_under`, `close_soc`, `self_ev_consumption` and `extra_self_production`. Penalties are negative. version: int, default: 1 Reward definition version. * ``1``: the reward of CityLearn 3.6.0 and earlier. A building with chargers gets only its EV terms; the base reward scales them but is not added. A building without chargers gets 0. With a central agent, the scale uses the district total of the base reward. Leaving with a state of charge 0.25 or more below the required departure SOC adds ``weights['soc_under'] ** 2`` (+25 with the default weights), which exceeds the moderate deficit term ``2 * weights['soc_under']`` (-10): severe under-charging is rewarded. * ``2``: every building gets its base reward plus its EV terms; a building without chargers gets its base reward. The scale uses the building's own base reward, with or without a central agent. A deficit of 0.25 or more adds ``min(2 * w, -(w ** 2))`` with ``w = weights['soc_under']`` (-25 with the default weights), never less than the moderate deficit term. Version 1 is the default so that existing schemas and trained policies keep their reward; use version 2 for new work. """ VERSIONS: Tuple[int, ...] = (1, 2) # Class default for instances pickled before the version attribute existed (CityLearn 3.6.0 and earlier). version: int = 1 def __init__(self, env_metadata: Mapping[str, Any], weights: Mapping[str, float] = None, version: int = None): super().__init__(env_metadata) self.version = self._parse_version(version, self.VERSIONS, type(self).__name__) # Default tunable weights for EV-related reward components self.weights = weights or { "no_car_charging": -5.0, "battery_limits": -2.0, "soc_impossible": -10.0, "soc_under": -5.0, "close_soc": 10.0, "self_ev_consumption": 5.0, "extra_self_production": 5.0, } self._last_base_reward_total = 0.0 self._last_penalty_total = 0.0 self._last_base_rewards_per_building: List[float] = [] self._last_ev_terms_per_building: List[float] = [] self._last_penalties_per_building: List[float] = [] def _charging_constraint_penalty(self, o: Mapping[str, Union[int, float, dict]]) -> float: violation = float(o.get("charging_constraint_violation_kwh", 0.0) or 0.0) return violation * self.charging_constraint_penalty_coefficient if violation > 0.0 else 0.0
[docs] def calculate(self, observations: List[Mapping[str, Union[int, float, dict]]]) -> List[float]: if self.version == 1: return self._calculate_version_1(observations) base_rewards = [float(v) for v in self._building_rewards(observations)] ev_terms = [] penalty_values = [] reward_list = [] for o, base_reward in zip(observations, base_rewards): ev_term = self.calculate_ev_penalty(o, base_reward) if o.get("electric_vehicles_chargers_dict", {}) else 0.0 penalty = self._charging_constraint_penalty(o) ev_terms.append(ev_term) penalty_values.append(penalty) reward_list.append(base_reward + ev_term - penalty) self._last_base_rewards_per_building = base_rewards self._last_ev_terms_per_building = ev_terms self._last_penalties_per_building = penalty_values self._last_base_reward_total = sum(base_rewards) if self.central_agent else base_rewards self._last_penalty_total = sum(penalty_values) if self.central_agent else penalty_values total_reward = [sum(reward_list)] if self.central_agent else reward_list LOGGER.debug(f"Calculated EV reward: {total_reward}") return total_reward
def _calculate_version_1(self, observations: List[Mapping[str, Union[int, float, dict]]]) -> List[float]: # In version 1 the "base rewards" diagnostics hold the EV terms, which are the whole reward before the # charging constraint penalty. current_reward = super().calculate(observations) reward_list = [] base_rewards = [] penalty_values = [] for i, o in enumerate(observations): ev_info = o.get("electric_vehicles_chargers_dict", {}) if not ev_info: reward = 0 else: if self.central_agent: reward_value = current_reward[0] if isinstance(current_reward, list) else current_reward reward = self.calculate_ev_penalty(o, reward_value) else: reward = self.calculate_ev_penalty(o, current_reward[i]) base_rewards.append(reward) violation = float(o.get("charging_constraint_violation_kwh", 0.0) or 0.0) penalty = violation * self.charging_constraint_penalty_coefficient if violation > 0.0 else 0.0 penalty_values.append(penalty) reward -= penalty reward_list.append(reward) self._last_base_rewards_per_building = base_rewards self._last_penalties_per_building = penalty_values total_reward = [sum(reward_list, numpy1_scalar(0))] if self.central_agent else reward_list self._last_base_reward_total = sum(base_rewards) if self.central_agent else base_rewards self._last_penalty_total = sum(penalty_values) if self.central_agent else penalty_values LOGGER.debug(f"Calculated EV reward: {total_reward}") return total_reward
[docs] def get_required_observation_names(self) -> Optional[Iterable[str]]: return ( 'net_electricity_consumption', 'electric_vehicles_chargers_dict', 'charging_constraint_violation_kwh', )
[docs] def calculate_ev_penalty(self, o: Mapping[str, Union[int, float, dict]], current_reward: float) -> float: penalty_total = 0.0 ev_chargers: dict = o.get("electric_vehicles_chargers_dict", {}) net_energy_before = o.get("net_electricity_consumption", 0) # Bounding the multiplier to avoid extreme scaling penalty_multiplier = 1.0 / (1.0 + abs(current_reward)) for charger_id, data in ev_chargers.items(): contributions = {k: 0.0 for k in self.weights.keys()} if not data["connected"]: last_charged_kwh = data.get("last_charged_kwh", 0.0) or 0.0 if abs(last_charged_kwh) > 0.1: contributions["no_car_charging"] += self.weights["no_car_charging"] * penalty_multiplier LOGGER.debug(f"Charger {charger_id} | EV not connected | Contributions: {contributions}") penalty_total += sum(contributions.values()) continue # Extract values soc_prev = data.get("previous_battery_soc") soc_now = data.get("battery_soc") capacity = data.get("battery_capacity") min_capacity = data.get("min_capacity") last_charged_kwh = data.get("last_charged_kwh", 0) if last_charged_kwh is None: last_charged_kwh = 0.0 required_soc = data.get("required_soc") hours_until_departure = data.get("hours_until_departure", 0) max_charging_power = data.get("max_charging_power", 0) max_discharging_power = data.get("max_discharging_power", 0) if soc_prev is None or soc_now is None or capacity is None: raise ValueError("Something went wrong, this values should not be none") continue # Battery limits current_energy = soc_prev * numpy1_scalar(capacity) + last_charged_kwh if current_energy > capacity or current_energy < min_capacity: contributions["battery_limits"] += self.weights["battery_limits"] * penalty_multiplier # SoC penalties/rewards if required_soc is not None and hours_until_departure >= 0: soc_diff = soc_now - required_soc soc_diff_kWh = soc_diff * numpy1_scalar(capacity) max_possible_charge = max(max_charging_power, 0.0) * max(hours_until_departure, 0.0) max_possible_discharge = max(max_discharging_power, 0.0) * max(hours_until_departure, 0.0) soc_deficit_kwh = max(required_soc - soc_now, 0.0) * numpy1_scalar(capacity) if soc_deficit_kwh > max_possible_charge: contributions["soc_impossible"] += self.weights["soc_impossible"] * penalty_multiplier if hours_until_departure == 0: if -0.25 < soc_diff <= -0.10: contributions["soc_under"] += 2 * self.weights["soc_under"] * penalty_multiplier elif soc_diff <= -0.25: weight = self.weights["soc_under"] # Version 1 squares the signed weight, which turns the penalty into a bonus. severe = (weight ** 2) if self.version == 1 else min(2.0 * weight, -(weight ** 2)) contributions["soc_under"] += severe * penalty_multiplier elif -0.10 < soc_diff <= 0.10: contributions["close_soc"] += self.weights["close_soc"] * penalty_multiplier if abs(soc_diff_kWh) <= max(max_possible_charge, max_possible_discharge): reward_multiplier = 1.0 / (hours_until_departure + 0.1) contributions["close_soc"] += self.weights["close_soc"] * penalty_multiplier * reward_multiplier # Self-production reward if last_charged_kwh > 0 and net_energy_before < 0: contributions["extra_self_production"] += self.weights["extra_self_production"] * penalty_multiplier elif last_charged_kwh < 0 and net_energy_before < 0: contributions["extra_self_production"] += -0.5 * self.weights["extra_self_production"] * penalty_multiplier # Self-consumption reward if last_charged_kwh < 0 and net_energy_before > 0: contributions["self_ev_consumption"] += self.weights["self_ev_consumption"] * penalty_multiplier elif last_charged_kwh > 0 and net_energy_before > 0: contributions["self_ev_consumption"] += -0.5 * self.weights["self_ev_consumption"] * penalty_multiplier LOGGER.debug(f"Charger {charger_id} | Contributions: {contributions}") penalty_total += sum(contributions.values()) return penalty_total
[docs] class OutageUnservedEnergyReward(RewardFunction): r"""Base reward plus a penalty on energy left unserved during power outages. During a power outage a building cannot draw from the grid, so loads that its PV and storage cannot cover go unserved. Rewards built on net grid consumption see zero consumption at those time steps and do not penalize the shortfall. This reward adds that penalty to a base reward: .. math:: r = r_{\textrm{base}} - \lambda \sum_{i} u_i where :math:`u_i` is the `power_outage_unserved_energy` observation of building :math:`i` and :math:`\lambda` is `coefficient`. That observation uses the same definition as the `power_outage_normalized_unserved_energy_total` KPI and is zero at time steps without an outage. With a central agent the per-building terms are summed. Parameters ---------- env_metadata: Mapping[str, Any]: General static information about the environment. coefficient: float, default: 1.0 Penalty per kWh of unserved energy, in the units of the base reward. base_reward: Union[RewardFunction, str, Mapping[str, Any]], optional Reward the penalty is added to: an instance, an import path such as `'citylearn.reward_function.ComfortReward'`, or a mapping with `type` and optional `attributes` keys as in the schema. Defaults to :py:class:`RewardFunction`. Passing another `OutageUnservedEnergyReward` applies the penalty twice. """ def __init__( self, env_metadata: Mapping[str, Any], coefficient: float = None, base_reward: Union[RewardFunction, str, Mapping[str, Any]] = None, **kwargs ): self.base_reward = self._build_base_reward(base_reward, env_metadata) self.coefficient = 1.0 if coefficient is None else float(coefficient) super().__init__(env_metadata, **kwargs) @staticmethod def _build_base_reward(base_reward, env_metadata) -> RewardFunction: if base_reward is None: return RewardFunction(env_metadata) if isinstance(base_reward, RewardFunction): return base_reward if isinstance(base_reward, str): base_reward = {'type': base_reward} if isinstance(base_reward, Mapping) and isinstance(base_reward.get('type'), str): module_name, _, class_name = base_reward['type'].rpartition('.') constructor = getattr(importlib.import_module(module_name), class_name) return constructor(env_metadata, **(base_reward.get('attributes') or {})) raise TypeError( 'base_reward must be a RewardFunction, an import path or a mapping with a "type" import path, ' f'got {base_reward!r}.' ) @property def env_metadata(self) -> Mapping[str, Any]: return self._env_metadata @env_metadata.setter def env_metadata(self, env_metadata: Mapping[str, Any]): self._env_metadata = env_metadata base_reward = getattr(self, 'base_reward', None) if base_reward is not None: base_reward.env_metadata = env_metadata
[docs] def reset(self): self.base_reward.reset()
[docs] def get_required_observation_names(self) -> Optional[Iterable[str]]: names = self.base_reward.get_required_observation_names() if names is None: return None return tuple(dict.fromkeys([*names, 'power_outage_unserved_energy']))
[docs] def calculate(self, observations: List[Mapping[str, Union[int, float]]]) -> List[float]: base = np.asarray(self.base_reward.calculate(observations), dtype='float64').reshape(-1) penalty = self.coefficient*np.array( [float(o.get('power_outage_unserved_energy', 0.0) or 0.0) for o in observations], dtype='float64' ) if self.central_agent: reward = [float(base.sum() - penalty.sum())] else: reward = (base - penalty).tolist() return reward