// ==++==
//
// Copyright (c) Microsoft Corporation. All rights reserved.
//
// ==--==
// =+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+
//
// SearchAlgorithms.h
//
// Header file containing definitions for all scheduling algorithms.
//
// =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
namespace Concurrency
{
namespace details
{
///
/// Variant type representing a work item returned from a search.
///
class WorkItem
{
public:
///
/// The type of work item.
///
enum WorkItemType
{
//
// Specific types:
//
// Empty work item
WorkItemTypeNone = 0x0,
// Work item is a context
WorkItemTypeContext = 0x1,
// Work item is a realized chore. Bind() will associate to a context if a context can be allocated
WorkItemTypeRealizedChore = 0x2,
// Work item is an unrealized chore. Bind() will associate to a context if a context can be allocated
WorkItemTypeUnrealizedChore = 0x4,
// Work item is a token to an realized chore. ResolveToken() will grab the item if it is still available.
WorkItemTypeRealizedChoreToken = 0x8,
// Work item is a token to an unrealized chore. ResolveToken() will grab the item if it is still available.
WorkItemTypeUnrealizedChoreToken = 0x10,
//
// General Masks:
//
WorkItemTypeMaskAnyRealizedChore = 0xA,
WorkItemTypeMaskAnyUnrealizedChore = 0x14
};
///
/// Default constructor for a work item.
///
WorkItem() :
m_type(WorkItemTypeNone),
m_pItem(NULL)
{
}
///
/// Constructs a work item from an internal context.
///
WorkItem(InternalContextBase *pContext);
///
/// Constructs a work item from a realized chore.
///
WorkItem(RealizedChore *pRealizedChore, ScheduleGroupSegmentBase *pSegment) :
m_type(WorkItemTypeRealizedChore),
m_pSegment(pSegment),
m_pRealizedChore(pRealizedChore)
{
}
///
/// Constructs a work item from an unrealized chore.
///
WorkItem(_UnrealizedChore *pUnrealizedChore, ScheduleGroupSegmentBase *pSegment) :
m_type(WorkItemTypeUnrealizedChore),
m_pSegment(pSegment),
m_pUnrealizedChore(pUnrealizedChore)
{
}
///
/// Constructs a work item from an unrealized chore token.
///
WorkItem(WorkQueue *pWorkQueue, ScheduleGroupSegmentBase *pSegment) :
m_type(WorkItemTypeUnrealizedChoreToken),
m_pSegment(pSegment),
m_pWorkQueue(pWorkQueue)
{
}
///
/// Constructs a work item from a realized chore token.
///
WorkItem(ScheduleGroupSegmentBase *pSegment) :
m_type(WorkItemTypeRealizedChoreToken),
m_pSegment(pSegment),
m_pRealizedChore(nullptr)
{
}
///
/// Transfers reference counts as necessary to inline the given work item on the given context. This may
/// only be called on a work item that can be inlined (e.g.: an unbound one).
///
///
/// The context that is attempting to inline the work item.
///
void TransferReferences(InternalContextBase *pContext);
///
/// Resolves a token to an underlying work item.
///
bool ResolveToken();
///
/// Binds the work item to a context and returns the context. This may or may not allocate a new context. Note that
/// act of binding which performs a context allocation will transfer a single count of work to the counter of the new
/// context.
///
InternalContextBase *Bind();
///
/// Binds the work item to the specified context (which is allocated). This will never allocate a new context.
///
void BindTo(InternalContextBase *pContext);
///
/// Invokes the work item.
///
void Invoke();
///
/// Accessor for type.
///
WorkItemType GetType() const
{
return m_type;
}
///
/// Returns the work item.
///
void *GetItem() const
{
return m_pItem;
}
///
/// Returns whether the work item is a context or not.
///
bool IsContext() const
{
return (m_type == WorkItemTypeContext);
}
///
/// Returns whether the work item is a token or not.
///
bool IsToken() const
{
return ((m_type & (WorkItemTypeRealizedChoreToken | WorkItemTypeUnrealizedChoreToken)) != 0);
}
///
/// Accessor for a context.
///
InternalContextBase *GetContext() const
{
CONCRT_COREASSERT(m_type == WorkItemTypeContext);
return m_pContext;
}
///
/// Accessor for a realized chore.
///
RealizedChore *GetRealizedChore() const
{
CONCRT_COREASSERT(m_type == WorkItemTypeRealizedChore);
return m_pRealizedChore;
}
///
/// Accessor for an unrealized chore.
///
_UnrealizedChore *GetUnrealizedChore() const
{
CONCRT_COREASSERT(m_type == WorkItemTypeUnrealizedChore);
return m_pUnrealizedChore;
}
///
/// Accessor for the schedule group segment.
///
ScheduleGroupSegmentBase *GetScheduleGroupSegment() const
{
return m_pSegment;
}
///
/// Accessor for the schedule group.
///
ScheduleGroupBase *GetScheduleGroup() const;
private:
// The type of work item
WorkItemType m_type;
// The schedule group that the work item was found in.
ScheduleGroupSegmentBase *m_pSegment{};
// The work item itself
union
{
void *m_pItem;
WorkQueue *m_pWorkQueue;
InternalContextBase *m_pContext;
RealizedChore *m_pRealizedChore;
_UnrealizedChore *m_pUnrealizedChore;
};
};
///
/// A class which tracks iterator state for a search-for-work. This is generic in terms of search algorithm.
///
class WorkSearchContext
{
public:
///
/// Describes the search algorithm being utilized.
///
enum Algorithm
{
AlgorithmNotSet = 0,
AlgorithmCacheLocal,
AlgorithmFair
};
///
/// Describes the type of affinity we are allowed to search for.
///
enum SearchAffinity
{
// Search for non-affine work within the domain of the search.
SearchNonAffine,
// Search for work which is affine to the search context within the domain of the search.
SearchAffineLocal,
// Search for work which is affine to something OTHER than the search context within the domain of the search.
SearchAffineNotMe
};
///
/// Constructs a work search context that will be reset later.
///
WorkSearchContext() : m_pVirtualProcessor(NULL), m_pScheduler(NULL), m_pSearchFn(NULL), m_pSearchYieldFn(NULL)
{
}
///
/// Constructs a work search context (an iterator position for a search algorithm).
///
WorkSearchContext(VirtualProcessor *pVirtualProcessor, Algorithm algorithm)
{
Reset(pVirtualProcessor, algorithm);
}
///
/// Resets the work search context to utilize the specified algorithm at the starting iterator position.
///
///
/// The virtual processor binding the searching.
///
///
/// What algorithm to reset the iterator with.
///
void Reset(VirtualProcessor *pVirtualProcessor, Algorithm algorithm);
///
/// Searches from the last iterator position according to the set algorithm. This can return any type of work
/// item (context, realized chore, or unrealized chore)
///
///
/// Upon successful return, the resulting work item is placed here along with information about what group it was found in, etc...
///
///
/// The schedule group segment of the context which is performing the search.
///
///
/// An indication as to whether this is a last pass SFW.
///
///
/// What type of work items are allowed to be returned.
///
///
/// An indication of whether a work item was found or not.
///
bool Search(WorkItem *pWorkItem,
ScheduleGroupSegmentBase *pOriginSegment,
bool fLastPass = false,
ULONG allowableTypes = WorkItem::WorkItemTypeContext | WorkItem::WorkItemTypeRealizedChore | WorkItem::WorkItemTypeUnrealizedChore)
{
return (this->*m_pSearchFn)(pWorkItem, pOriginSegment, fLastPass, allowableTypes);
}
///
/// Searches from the last iterator position according to the set algorithm for a yield. This can return any type of
/// work item (context, realized chore, or unrealized chore)
///
bool YieldingSearch(WorkItem *pWorkItem,
ScheduleGroupSegmentBase *pOriginSegment,
bool fLastPass = false,
ULONG allowableTypes = WorkItem::WorkItemTypeContext | WorkItem::WorkItemTypeRealizedChore)
{
return (this->*m_pSearchYieldFn)(pWorkItem, pOriginSegment, fLastPass, allowableTypes);
}
private:
// **************************************************
// Common:
//
///
/// Describes where the bias towards certain lists is at within SFW.
///
enum BiasStageType
{
BiasStageNone,
BiasStageFlipLRC,
BiasStageSkipLRC
};
// The virtual processor binding the search.
VirtualProcessor *m_pVirtualProcessor;
// The scheduler
SchedulerBase *m_pScheduler;
// The mask ID of the virtual processor binding the search.
unsigned int m_maskId{};
// How many times work has been found in the LRC since the last reset.
ULONG m_LRCBias{};
// The service time stamp
ULONGLONG m_serviceTick{};
// TRANSITION: This goes with real priority...
ULONGLONG m_lastPriorityPull{};
// The search function to utilize.
bool (WorkSearchContext::*m_pSearchFn)(WorkItem *pWorkItem,
ScheduleGroupSegmentBase *pOriginSegment,
bool fLastPass,
ULONG allowableTypes);
// The search function to utilize for yielding.
bool (WorkSearchContext::*m_pSearchYieldFn)(WorkItem *pWorkItem,
ScheduleGroupSegmentBase *pOriginSegment,
bool fLastPass,
ULONG allowableTypes);
//
// TRANSITION: This goes with real priority...
//
bool CheckPriorityList(ULONGLONG serviceTime)
{
bool fCheck = ((serviceTime - m_lastPriorityPull) > (ULONGLONG)1000);
if (fCheck)
m_lastPriorityPull = serviceTime;
return fCheck;
}
///
/// Performs a quick search of a particular segment.
///
bool QuickSearch(ScheduleGroupSegmentBase *pQCSegment,
WorkItem *pWorkItem,
bool fLastPass,
ULONG allowableTypes);
///
/// Performs a quick yielding search of a particular segment.
///
bool QuickSearchYield(ScheduleGroupSegmentBase *pQCSegment,
WorkItem *pWorkItem,
bool fLastPass,
ULONG allowableTypes);
///
/// Performs a pre-search for any "special" contexts (e.g.: the UMS SUT)
///
bool PreSearch(WorkItem *pWorkItem);
///
/// Steals a local runnable from a virtual processor within the specified node. Note that this allows a given virtual processor
/// to be skipped.
///
bool StealLocalRunnable(WorkItem *pWorkItem, SchedulingNode *pNode, VirtualProcessor *pSkipVirtualProcessor);
///
/// Steals a local runnable from a virtual processor of any scheduling node other than the specified local node.
///
bool StealForeignLocalRunnable(WorkItem *pWorkItem, SchedulingNode *pLocalNode);
///
/// Gets a local runnable context from the specified virtual processor.
///
bool GetLocalRunnable(WorkItem *pWorkItem, VirtualProcessor *pVirtualProcessor, bool fYieldingSearch);
///
/// Gets a runnable from the specified schedule group segment.
///
///
/// If a work item is found, the work item will be returned here.
///
///
/// The schedule group segment in which to look for a runnable context.
///
///
/// An indication of whether or not a runnable context was found in the segment.
///
bool GetRunnableContext(WorkItem *pWorkItem, ScheduleGroupSegmentBase *pSegment);
///
/// Gets a realized chore from the specified schedule group segment.
///
///
/// If a work item is found, the work item will be returned here.
///
///
/// The schedule group segment in which to look for a realized chore.
///
///
/// If true, the actual work item is returned. If false, a token to the work is returned. The work is not dequeued until the token
/// is resolved.
///
///
/// An indication of whether or not a realized chore was found in the segment.
///
bool GetRealizedChore(WorkItem *pWorkItem, ScheduleGroupSegmentBase *pSegment, bool fRealWork);
///
/// Gets an unrealized chore from the specified schedule group segment.
///
///
/// If a work item is found, the work item will be returned here.
///
///
/// The schedule group segment in which to look for an unrealized chore.
///
///
/// Whether to steal the task at the bottom end of the work stealing queue even if it is an affinitized to a location
/// that has active searches. This is set to true on the final SFW pass to ensure a vproc does not deactivate while there
/// are chores higher up in the queue that are un-affinitized and therefore inaccessible via a mailbox.
///
///
/// If true, the actual work item is returned. If false, a token to the work is returned. The work is not dequeued until the token
/// is resolved.
///
///
/// An indication of whether or not an unrealized chore was found in the segment.
///
bool GetUnrealizedChore(WorkItem *pWorkItem, ScheduleGroupSegmentBase *pSegment, bool fForceStealLocalized, bool fRealWork);
///
/// Determines if a segment should be skipped given the search parameters and the segment's affinity.
///
///
/// The segment to query about skipping.
///
///
/// A segment which should be arbitrarily skipped regardless of affinity type. This parameter can be NULL.
///
///
/// The search affinity type to query for.
///
///
/// An indication as to whether this is a last pass SFW.
///
///
/// An indication as to whether pSegment should be skipped according to the pSkipSegment and affinity parameters.
///
bool SkipSegmentSearch(ScheduleGroupSegmentBase *pSegment, ScheduleGroupSegmentBase *pSkipSegment, SearchAffinity affinity, bool fLastPass);
///
/// Searches the schedule group to which pSegment belongs to find a runnable. The group is searched for segments according to the specified
/// affinity type.
///
///
/// If an appropriate runnable is found, the resulting work item will be placed here.
///
///
/// A segment within the group to search. This segment has bias within the group if it matches the specified affinity type.
///
///
/// An indication as to whether this is a last pass SFW.
///
///
/// An indication of whether a work item was found or not.
///
bool GetRunnableContextWithinGroup(WorkItem *pWorkItem, ScheduleGroupSegmentBase *pSegment, SearchAffinity affinity, bool fLastPass);
///
/// Searches the schedule group to which pSegment belongs to find a realized chore. The group is searched for segments according to the specified
/// affinity type.
///
///
/// If an appropriate realized chore is found, the resulting work item will be placed here.
///
///
/// A segment within the group to search. This segment has bias within the group if it matches the specified affinity type.
///
///
/// An indication as to whether this is a last pass SFW.
///
///
/// An indication of whether a work item was found or not.
///
bool GetRealizedChoreWithinGroup(WorkItem *pWorkItem, ScheduleGroupSegmentBase *pSegment, bool fRealWork, SearchAffinity affinity, bool fLastPass);
///
/// Searches the schedule group to which pSegment belongs to find an unrealized chore. The group is searched for segments according to the
/// specified affinity type.
///
///
/// If an appropriate unrealized chore is found, the resulting work item will be placed here.
///
///
/// A segment within the group to search. This segment has bias within the group if it matches the specified affinity type.
///
///
/// An indication as to whether this is a last pass SFW.
///
///
/// An indication of whether a work item was found or not.
///
bool GetUnrealizedChoreWithinGroup(WorkItem *pWorkItem, ScheduleGroupSegmentBase *pSegment, bool fRealWork, SearchAffinity affinity, bool fLastPass);
///
/// Called on any biased work.
///
void LRCBias()
{
m_LRCBias++;
}
///
/// Resets the local bias counter but not the ring bias counter.
///
void ResetLRCBias()
{
m_LRCBias = 0;
}
///
/// Returns the current stage of local bias.
///
BiasStageType BiasStage()
{
if (m_LRCBias < 101)
return BiasStageNone; // (fwd) Normal --> LRC LIFO
else if (m_LRCBias < 127)
return BiasStageFlipLRC; // (fwd) Flip LRC --> LRC FIFO
else
return BiasStageSkipLRC; // (fwd) Skip LRC --> runnables
}
// **************************************************
// Cache Local Algorithm:
//
///
/// Searches for a runnable within the specified ring. Before searching elsewhere, it searches the segment and group specified by
/// pBiasSegment according to the rules of the search and the requested affinity type.
///
///
/// If a work item is found, the work item will be returned here.
///
///
/// The scheduling ring to search.
///
///
/// The segment to bias the search to. This segment and its corresponding group are searched first!
///
///
/// Determines whether or not to check other local LRCs in this search.
///
///
/// The search affinity type to query for.
///
///
/// A bitmap of work-types allowed to be returned from the search. This indicates whether or not runnables, realized chores, or unrealized chores
/// can be returned as well as whether the actual work item or only a token should be returned.
///
///
/// An indication as to whether this is a last pass SFW.
///
///
/// An indication of whether a runnable was found in the bias segment, group, or the specified ring.
///
bool SearchCacheLocal_Runnables(WorkItem *pWorkItem, SchedulingRing *pRing, ScheduleGroupSegmentBase *pBiasSegment,
bool fOtherLocalLRCCheck, SearchAffinity affinity, ULONG allowableTypes, bool fLastPass);
///
/// Searches for a realized chore within the specified ring. Before searching elsewhere, it searches the segment and group specified by
/// pBiasSegment according to the rules of the search and the requested affinity type.
///
///
/// If a work item is found, the work item will be returned here.
///
///
/// The scheduling ring to search.
///
///
/// The segment to bias the search to. This segment and its corresponding group are searched first!
///
///
/// If true, the actual work item is returned. If false, a token to the work is returned. The work is not dequeued until the token
/// is resolved.
///
///
/// The search affinity type to query for.
///
///
/// A bitmap of work-types allowed to be returned from the search. This indicates whether or not runnables, realized chores, or unrealized chores
/// can be returned as well as whether the actual work item or only a token should be returned.
///
///
/// An indication as to whether this is a last pass SFW.
///
///
/// An indication of whether a realized chore was found in the bias segment, group, or the specified ring.
///
bool SearchCacheLocal_Realized(WorkItem *pWorkItem, SchedulingRing *pRing, ScheduleGroupSegmentBase *pBiasSegment,
bool fRealWork, SearchAffinity affinity, ULONG allowableTypes, bool fLastPass);
///
/// Searches for an unrealized chore within the specified ring. Before searching elsewhere, it searches the segment and group specified by
/// pBiasSegment according to the rules of the search and the requested affinity type.
///
///
/// If a work item is found, the work item will be returned here.
///
///
/// The scheduling ring to search.
///
///
/// The segment to bias the search to. This segment and its corresponding group are searched first!
///
///
/// If true, the actual work item is returned. If false, a token to the work is returned. The work is not dequeued until the token
/// is resolved.
///
///
/// The search affinity type to query for.
///
///
/// A bitmap of work-types allowed to be returned from the search. This indicates whether or not runnables, realized chores, or unrealized chores
/// can be returned as well as whether the actual work item or only a token should be returned.
///
///
/// An indication as to whether this is a last pass SFW.
///
///
/// An indication of whether an unrealized chore was found in the bias segment, group, or the specified ring.
///
bool SearchCacheLocal_Unrealized(WorkItem *pWorkItem, SchedulingRing *pRing, ScheduleGroupSegmentBase *pBiasSegment,
bool fRealWork, SearchAffinity affinity, ULONG allowableTypes, bool fLastPass);
///
/// Searches for work within the scheduler according to the cache local (schedule group local) search algorithm.
///
///
/// If a work item is found, the work item will be returned here.
///
///
/// The segment to bias the search to.
///
///
/// An indication as to whether this is a last pass SFW.
///
///
/// A bitmap of work-types allowed to be returned from the search. This indicates whether or not runnables, realized chores, or unrealized chores
/// can be returned as well as whether the actual work item or only a token should be returned.
///
///
/// An indication of whether a work item was found or not.
///
bool SearchCacheLocal(WorkItem *pWorkItem, ScheduleGroupSegmentBase *pOriginSegment, bool fLastPass, ULONG allowableTypes);
///
/// Searches for work within the scheduler according to the cache local (schedule group local) search algorithm for yielding.
///
///
/// If a work item is found, the work item will be returned here.
///
///
/// The segment to bias the search to.
///
///
/// An indication as to whether this is a last pass SFW.
///
///
/// A bitmap of work-types allowed to be returned from the search. This indicates whether or not runnables, realized chores, or unrealized chores
/// can be returned as well as whether the actual work item or only a token should be returned.
///
///
/// An indication of whether a work item was found or not.
///
bool SearchCacheLocalYield(WorkItem *pWorkItem, ScheduleGroupSegmentBase *pOriginSegment, bool fLastPass, ULONG allowableTypes);
// **************************************************
// Fair Algorithm:
//
///
/// Performs a fair search for runnables in the specified ring.
///
bool SearchFair_Runnables(WorkItem *pWorkItem, SchedulingRing *pRing);
///
/// Performs a fair search for realized chores in the specified ring.
///
bool SearchFair_Realized(WorkItem *pWorkItem, SchedulingRing *pRing, bool fRealItem);
///
/// Performs a fair search for unrealized chores in the specified ring.
///
bool SearchFair_Unrealized(WorkItem *pWorkItem, SchedulingRing *pRing, bool fRealItem);
///
/// Performs a fair search for work.
///
bool SearchFair(WorkItem *pWorkItem, ScheduleGroupSegmentBase *pOriginSegment, bool fLastPass, ULONG allowableTypes);
///
/// Performs a fair search for work in the yielding case.
///
bool SearchFairYield(WorkItem *pWorkItem, ScheduleGroupSegmentBase *pOriginSegment, bool fLastPass, ULONG allowableTypes);
};
}
}