in src/videodownlinkbandwidthpolicy/VideoPriorityBasedPolicy.ts [342:564]
protected calculateOptimalReceiveStreams(): void {
const chosenStreams: VideoStreamDescription[] = [];
const remoteInfos: VideoStreamDescription[] = this.videoIndex.remoteStreamDescriptions();
if (remoteInfos.length === 0 || this.videoPreferences?.isEmpty()) {
this.optimalReceiveStreams = [];
return;
}
const lastProbeState = this.rateProbeState;
this.cleanBwPausedTiles(remoteInfos);
this.handleAppPausedStreams(chosenStreams, remoteInfos);
const attendeeIdToPreference = new Map<string, VideoPreference>();
for (const preference of this.getCurrentVideoPreferences()) {
attendeeIdToPreference.set(preference.attendeeId, preference);
}
// We can pre-emptively filter to just remote stream descriptions for remote video sources we already
// have configuration for. We do not care about streams we aren't going to subscribe to.
const remoteInfosWithPreference = remoteInfos.filter((stream: VideoStreamDescription) => {
return attendeeIdToPreference.has(stream.attendeeId);
});
const sameStreamChoices = !this.streamsWithPreferenceDidChange(remoteInfosWithPreference);
const noMajorChange = !this.startupPeriod && sameStreamChoices;
// If no major changes then don't allow subscribes for the allowed amount of time
if (
noMajorChange &&
Date.now() - this.lastSubscribeTimestamp < this.timeBeforeAllowSubscribeMs
) {
return;
}
// Sort streams by bitrate ascending.
remoteInfosWithPreference.sort((a, b) => {
if (a.maxBitrateKbps === b.maxBitrateKbps) {
return a.streamId - b.streamId;
}
return a.maxBitrateKbps - b.maxBitrateKbps;
});
// Convert 0 avg bitrates to max, handle special cases, and remove upgrades that downgrade resolution or framerate
for (const info of remoteInfosWithPreference) {
if (info.avgBitrateKbps === 0 || info.avgBitrateKbps > info.maxBitrateKbps) {
// Content can be a special case
if (info.attendeeId.endsWith(ContentShareConstants.Modality) && info.maxBitrateKbps < 100) {
info.maxBitrateKbps = info.avgBitrateKbps;
} else {
info.avgBitrateKbps = info.maxBitrateKbps;
}
}
}
const streamIdsToRemove = new Set<number>();
for (const preference of this.getCurrentVideoPreferences()) {
const streamsForAttendees = remoteInfosWithPreference.filter(
(description: VideoStreamDescription) => {
return description.attendeeId === preference.attendeeId;
}
);
if (streamsForAttendees.length < 3) {
// If `streamsForAttendees.length` is 0, they are stale preference for remote video that no longer exists
// if 1 or 2, there is nothing to filter out
continue;
}
let maxFrameRate = 15;
let maxPixels = 320 * 480;
streamsForAttendees.forEach((stream: VideoStreamDescription) => {
maxFrameRate = Math.max(maxFrameRate, stream.maxFrameRate);
maxPixels = Math.max(maxPixels, stream.totalPixels());
});
streamsForAttendees.sort((a: VideoStreamDescription, b: VideoStreamDescription) => {
if (
preference.degradationPreference === VideoQualityAdaptationPreference.MaintainResolution
) {
// This may seem counter-intuitive but given we want to upgrade resolution first, and framerate
// last, we want to sort by framerate first, and resolution only if framerate is close.
//
// That way, e.g. the first three streams will all contain increases in resolution. We will skip any down
// grades in resolution (i.e. to hop to a higher framerate stream) in the section below
if (Math.abs(a.maxFrameRate - b.maxFrameRate) < 2) {
return a.totalPixels() - b.totalPixels();
}
return a.maxFrameRate - b.maxFrameRate;
} else if (
preference.degradationPreference === VideoQualityAdaptationPreference.MaintainFramerate
) {
// See note above, but swap resolution for framerate
if (a.totalPixels() === b.totalPixels()) {
return a.maxFrameRate - b.maxFrameRate;
}
return a.totalPixels() - b.totalPixels();
} else {
// In 'balanced' mode, we want a slight preference towards upgrading resolution first
// when moving up the bitrate ladder, so we weigh the bitrate by a framerate
// and resolution derived value that is tuned to be reasonable in most cases. This attempts to
// mimic the diagrams in the guide.
//
// A higher constant makes the policy prefer resolution to framerate more at lower values
const framerateExponentConstant = 5;
// A higher constant here makes the policy prefer framerate to resolution more at higher values
const resolutionBaseConstant = 2;
const weighByFramerate = (stream: VideoStreamDescription): number => {
return (
stream.avgBitrateKbps *
Math.pow(
2,
((framerateExponentConstant * stream.maxFrameRate) / maxFrameRate) *
Math.pow(resolutionBaseConstant * 2, stream.totalPixels() / maxPixels)
)
);
};
return weighByFramerate(a) - weighByFramerate(b);
}
});
let lastInfo = undefined;
// Downgrades during recovery are unusual and jarring. Here we will strip out any (these should result in paths that
// match those in the priority downlink policy guide).
for (const info of streamsForAttendees) {
if (lastInfo !== undefined) {
if (
(info.maxFrameRate < lastInfo.maxFrameRate &&
Math.abs(lastInfo.maxFrameRate - info.maxFrameRate) > 2) ||
info.totalPixels() < lastInfo.totalPixels()
) {
streamIdsToRemove.add(info.streamId);
continue;
}
}
lastInfo = info;
}
}
const filteredRemoteAttendeesWithPreference = remoteInfosWithPreference.filter(
(info: VideoStreamDescription) => {
return !streamIdsToRemove.has(info.streamId);
}
);
this.logger.info(`Filtered to ${JSON.stringify(filteredRemoteAttendeesWithPreference)}`);
const rates: PolicyRates = {
targetDownlinkBitrate: 0,
chosenTotalBitrate: 0,
deltaToNextUpgrade: 0,
};
rates.targetDownlinkBitrate = this.determineTargetRate();
const numberOfParticipants = this.subscribedReceiveSet.size();
const currentEstimated = this.downlinkStats.bandwidthEstimateKbps;
// Use videoPriorityBasedPolicyConfig to add additional delays based on network conditions
const dontAllowSubscribe = !this.videoPriorityBasedPolicyConfig.allowSubscribe(
numberOfParticipants,
currentEstimated
);
// `priorityPolicy` assumes that we have sorted `filteredRemoteAttendeesWithPreference` in order of quality
if (this.probeFailed) {
// When probe failed, we set timeBeforeAllowSubscribeMs to 3x longer
// Since we have passed the subscribe interval now, we will try to probe again
this.probeFailed = false;
// For the same reason above, reset time before allow subscribe to default
this.timeBeforeAllowSubscribeMs = VideoPriorityBasedPolicy.MIN_TIME_BETWEEN_SUBSCRIBE_MS;
if (noMajorChange && dontAllowSubscribe) return;
}
const upgradeStream: VideoStreamDescription = this.priorityPolicy(
rates,
filteredRemoteAttendeesWithPreference,
chosenStreams
);
const skipProbe = !serverSideNetworkAdaptionIsNoneOrDefault(this.serverSideNetworkAdaption);
let subscriptionChoice = UseReceiveSet.NewOptimal;
// Look for probing or override opportunities
if (!skipProbe && !this.startupPeriod && sameStreamChoices) {
if (this.rateProbeState === RateProbeState.Probing) {
subscriptionChoice = this.handleProbe(chosenStreams, rates.targetDownlinkBitrate);
} else if (rates.deltaToNextUpgrade !== 0) {
subscriptionChoice = this.maybeOverrideOrProbe(chosenStreams, rates, upgradeStream);
}
} else {
// If there was a change in streams to choose from, then cancel any probing or upgrades
this.setProbeState(RateProbeState.NotProbing);
this.lastUpgradeRateKbps = 0;
}
this.previousStreamsWithPreference = remoteInfosWithPreference;
this.videoPreferencesUpdated = false;
if (subscriptionChoice === UseReceiveSet.PreviousOptimal) {
this.logger.info(`bwe: keepSameSubscriptions stats:${JSON.stringify(this.downlinkStats)}`);
this.prevTargetRateKbps = rates.targetDownlinkBitrate;
return;
}
if (subscriptionChoice === UseReceiveSet.PreProbe) {
const subscribedRate = this.calculateSubscribeRate(this.preProbeNonPausedReceiveStreams);
this.optimalReceiveStreams = this.preProbeReceiveStreams.slice();
this.processBwPausedStreams(remoteInfosWithPreference, this.preProbeNonPausedReceiveStreams);
this.logger.info('bwe: Use Pre-Probe subscription subscribedRate:' + subscribedRate);
return;
}
this.optimalNonPausedReceiveStreams = chosenStreams.slice();
const lastNumberPaused = this.pausedBwAttendeeIds.size;
this.processBwPausedStreams(remoteInfosWithPreference, chosenStreams);
if (
this.logger.getLogLevel() <= LogLevel.INFO &&
(this.logCount % 15 === 0 ||
this.rateProbeState !== lastProbeState ||
this.optimalReceiveStreams.length !== chosenStreams.length ||
lastNumberPaused !== this.pausedBwAttendeeIds.size)
) {
this.logger.info(this.policyStateLogStr(remoteInfos, rates.targetDownlinkBitrate));
this.logCount = 0;
}
this.logCount++;
this.prevTargetRateKbps = rates.targetDownlinkBitrate;
this.optimalReceiveStreams = chosenStreams.slice();
}