// ==++== // // Copyright (c) Microsoft Corporation. All rights reserved. // // ==--== // =+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+ // // SearchAlgorithms.h // // Header file containing definitions for all scheduling algorithms. // // =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- namespace Concurrency { namespace details { /// /// Variant type representing a work item returned from a search. /// class WorkItem { public: /// /// The type of work item. /// enum WorkItemType { // // Specific types: // // Empty work item WorkItemTypeNone = 0x0, // Work item is a context WorkItemTypeContext = 0x1, // Work item is a realized chore. Bind() will associate to a context if a context can be allocated WorkItemTypeRealizedChore = 0x2, // Work item is an unrealized chore. Bind() will associate to a context if a context can be allocated WorkItemTypeUnrealizedChore = 0x4, // Work item is a token to an realized chore. ResolveToken() will grab the item if it is still available. WorkItemTypeRealizedChoreToken = 0x8, // Work item is a token to an unrealized chore. ResolveToken() will grab the item if it is still available. WorkItemTypeUnrealizedChoreToken = 0x10, // // General Masks: // WorkItemTypeMaskAnyRealizedChore = 0xA, WorkItemTypeMaskAnyUnrealizedChore = 0x14 }; /// /// Default constructor for a work item. /// WorkItem() : m_type(WorkItemTypeNone), m_pItem(NULL) { } /// /// Constructs a work item from an internal context. /// WorkItem(InternalContextBase *pContext); /// /// Constructs a work item from a realized chore. /// WorkItem(RealizedChore *pRealizedChore, ScheduleGroupSegmentBase *pSegment) : m_type(WorkItemTypeRealizedChore), m_pSegment(pSegment), m_pRealizedChore(pRealizedChore) { } /// /// Constructs a work item from an unrealized chore. /// WorkItem(_UnrealizedChore *pUnrealizedChore, ScheduleGroupSegmentBase *pSegment) : m_type(WorkItemTypeUnrealizedChore), m_pSegment(pSegment), m_pUnrealizedChore(pUnrealizedChore) { } /// /// Constructs a work item from an unrealized chore token. /// WorkItem(WorkQueue *pWorkQueue, ScheduleGroupSegmentBase *pSegment) : m_type(WorkItemTypeUnrealizedChoreToken), m_pSegment(pSegment), m_pWorkQueue(pWorkQueue) { } /// /// Constructs a work item from a realized chore token. /// WorkItem(ScheduleGroupSegmentBase *pSegment) : m_type(WorkItemTypeRealizedChoreToken), m_pSegment(pSegment), m_pRealizedChore(nullptr) { } /// /// Transfers reference counts as necessary to inline the given work item on the given context. This may /// only be called on a work item that can be inlined (e.g.: an unbound one). /// /// /// The context that is attempting to inline the work item. /// void TransferReferences(InternalContextBase *pContext); /// /// Resolves a token to an underlying work item. /// bool ResolveToken(); /// /// Binds the work item to a context and returns the context. This may or may not allocate a new context. Note that /// act of binding which performs a context allocation will transfer a single count of work to the counter of the new /// context. /// InternalContextBase *Bind(); /// /// Binds the work item to the specified context (which is allocated). This will never allocate a new context. /// void BindTo(InternalContextBase *pContext); /// /// Invokes the work item. /// void Invoke(); /// /// Accessor for type. /// WorkItemType GetType() const { return m_type; } /// /// Returns the work item. /// void *GetItem() const { return m_pItem; } /// /// Returns whether the work item is a context or not. /// bool IsContext() const { return (m_type == WorkItemTypeContext); } /// /// Returns whether the work item is a token or not. /// bool IsToken() const { return ((m_type & (WorkItemTypeRealizedChoreToken | WorkItemTypeUnrealizedChoreToken)) != 0); } /// /// Accessor for a context. /// InternalContextBase *GetContext() const { CONCRT_COREASSERT(m_type == WorkItemTypeContext); return m_pContext; } /// /// Accessor for a realized chore. /// RealizedChore *GetRealizedChore() const { CONCRT_COREASSERT(m_type == WorkItemTypeRealizedChore); return m_pRealizedChore; } /// /// Accessor for an unrealized chore. /// _UnrealizedChore *GetUnrealizedChore() const { CONCRT_COREASSERT(m_type == WorkItemTypeUnrealizedChore); return m_pUnrealizedChore; } /// /// Accessor for the schedule group segment. /// ScheduleGroupSegmentBase *GetScheduleGroupSegment() const { return m_pSegment; } /// /// Accessor for the schedule group. /// ScheduleGroupBase *GetScheduleGroup() const; private: // The type of work item WorkItemType m_type; // The schedule group that the work item was found in. ScheduleGroupSegmentBase *m_pSegment{}; // The work item itself union { void *m_pItem; WorkQueue *m_pWorkQueue; InternalContextBase *m_pContext; RealizedChore *m_pRealizedChore; _UnrealizedChore *m_pUnrealizedChore; }; }; /// /// A class which tracks iterator state for a search-for-work. This is generic in terms of search algorithm. /// class WorkSearchContext { public: /// /// Describes the search algorithm being utilized. /// enum Algorithm { AlgorithmNotSet = 0, AlgorithmCacheLocal, AlgorithmFair }; /// /// Describes the type of affinity we are allowed to search for. /// enum SearchAffinity { // Search for non-affine work within the domain of the search. SearchNonAffine, // Search for work which is affine to the search context within the domain of the search. SearchAffineLocal, // Search for work which is affine to something OTHER than the search context within the domain of the search. SearchAffineNotMe }; /// /// Constructs a work search context that will be reset later. /// WorkSearchContext() : m_pVirtualProcessor(NULL), m_pScheduler(NULL), m_pSearchFn(NULL), m_pSearchYieldFn(NULL) { } /// /// Constructs a work search context (an iterator position for a search algorithm). /// WorkSearchContext(VirtualProcessor *pVirtualProcessor, Algorithm algorithm) { Reset(pVirtualProcessor, algorithm); } /// /// Resets the work search context to utilize the specified algorithm at the starting iterator position. /// /// /// The virtual processor binding the searching. /// /// /// What algorithm to reset the iterator with. /// void Reset(VirtualProcessor *pVirtualProcessor, Algorithm algorithm); /// /// Searches from the last iterator position according to the set algorithm. This can return any type of work /// item (context, realized chore, or unrealized chore) /// /// /// Upon successful return, the resulting work item is placed here along with information about what group it was found in, etc... /// /// /// The schedule group segment of the context which is performing the search. /// /// /// An indication as to whether this is a last pass SFW. /// /// /// What type of work items are allowed to be returned. /// /// /// An indication of whether a work item was found or not. /// bool Search(WorkItem *pWorkItem, ScheduleGroupSegmentBase *pOriginSegment, bool fLastPass = false, ULONG allowableTypes = WorkItem::WorkItemTypeContext | WorkItem::WorkItemTypeRealizedChore | WorkItem::WorkItemTypeUnrealizedChore) { return (this->*m_pSearchFn)(pWorkItem, pOriginSegment, fLastPass, allowableTypes); } /// /// Searches from the last iterator position according to the set algorithm for a yield. This can return any type of /// work item (context, realized chore, or unrealized chore) /// bool YieldingSearch(WorkItem *pWorkItem, ScheduleGroupSegmentBase *pOriginSegment, bool fLastPass = false, ULONG allowableTypes = WorkItem::WorkItemTypeContext | WorkItem::WorkItemTypeRealizedChore) { return (this->*m_pSearchYieldFn)(pWorkItem, pOriginSegment, fLastPass, allowableTypes); } private: // ************************************************** // Common: // /// /// Describes where the bias towards certain lists is at within SFW. /// enum BiasStageType { BiasStageNone, BiasStageFlipLRC, BiasStageSkipLRC }; // The virtual processor binding the search. VirtualProcessor *m_pVirtualProcessor; // The scheduler SchedulerBase *m_pScheduler; // The mask ID of the virtual processor binding the search. unsigned int m_maskId{}; // How many times work has been found in the LRC since the last reset. ULONG m_LRCBias{}; // The service time stamp ULONGLONG m_serviceTick{}; // TRANSITION: This goes with real priority... ULONGLONG m_lastPriorityPull{}; // The search function to utilize. bool (WorkSearchContext::*m_pSearchFn)(WorkItem *pWorkItem, ScheduleGroupSegmentBase *pOriginSegment, bool fLastPass, ULONG allowableTypes); // The search function to utilize for yielding. bool (WorkSearchContext::*m_pSearchYieldFn)(WorkItem *pWorkItem, ScheduleGroupSegmentBase *pOriginSegment, bool fLastPass, ULONG allowableTypes); // // TRANSITION: This goes with real priority... // bool CheckPriorityList(ULONGLONG serviceTime) { bool fCheck = ((serviceTime - m_lastPriorityPull) > (ULONGLONG)1000); if (fCheck) m_lastPriorityPull = serviceTime; return fCheck; } /// /// Performs a quick search of a particular segment. /// bool QuickSearch(ScheduleGroupSegmentBase *pQCSegment, WorkItem *pWorkItem, bool fLastPass, ULONG allowableTypes); /// /// Performs a quick yielding search of a particular segment. /// bool QuickSearchYield(ScheduleGroupSegmentBase *pQCSegment, WorkItem *pWorkItem, bool fLastPass, ULONG allowableTypes); /// /// Performs a pre-search for any "special" contexts (e.g.: the UMS SUT) /// bool PreSearch(WorkItem *pWorkItem); /// /// Steals a local runnable from a virtual processor within the specified node. Note that this allows a given virtual processor /// to be skipped. /// bool StealLocalRunnable(WorkItem *pWorkItem, SchedulingNode *pNode, VirtualProcessor *pSkipVirtualProcessor); /// /// Steals a local runnable from a virtual processor of any scheduling node other than the specified local node. /// bool StealForeignLocalRunnable(WorkItem *pWorkItem, SchedulingNode *pLocalNode); /// /// Gets a local runnable context from the specified virtual processor. /// bool GetLocalRunnable(WorkItem *pWorkItem, VirtualProcessor *pVirtualProcessor, bool fYieldingSearch); /// /// Gets a runnable from the specified schedule group segment. /// /// /// If a work item is found, the work item will be returned here. /// /// /// The schedule group segment in which to look for a runnable context. /// /// /// An indication of whether or not a runnable context was found in the segment. /// bool GetRunnableContext(WorkItem *pWorkItem, ScheduleGroupSegmentBase *pSegment); /// /// Gets a realized chore from the specified schedule group segment. /// /// /// If a work item is found, the work item will be returned here. /// /// /// The schedule group segment in which to look for a realized chore. /// /// /// If true, the actual work item is returned. If false, a token to the work is returned. The work is not dequeued until the token /// is resolved. /// /// /// An indication of whether or not a realized chore was found in the segment. /// bool GetRealizedChore(WorkItem *pWorkItem, ScheduleGroupSegmentBase *pSegment, bool fRealWork); /// /// Gets an unrealized chore from the specified schedule group segment. /// /// /// If a work item is found, the work item will be returned here. /// /// /// The schedule group segment in which to look for an unrealized chore. /// /// /// Whether to steal the task at the bottom end of the work stealing queue even if it is an affinitized to a location /// that has active searches. This is set to true on the final SFW pass to ensure a vproc does not deactivate while there /// are chores higher up in the queue that are un-affinitized and therefore inaccessible via a mailbox. /// /// /// If true, the actual work item is returned. If false, a token to the work is returned. The work is not dequeued until the token /// is resolved. /// /// /// An indication of whether or not an unrealized chore was found in the segment. /// bool GetUnrealizedChore(WorkItem *pWorkItem, ScheduleGroupSegmentBase *pSegment, bool fForceStealLocalized, bool fRealWork); /// /// Determines if a segment should be skipped given the search parameters and the segment's affinity. /// /// /// The segment to query about skipping. /// /// /// A segment which should be arbitrarily skipped regardless of affinity type. This parameter can be NULL. /// /// /// The search affinity type to query for. /// /// /// An indication as to whether this is a last pass SFW. /// /// /// An indication as to whether pSegment should be skipped according to the pSkipSegment and affinity parameters. /// bool SkipSegmentSearch(ScheduleGroupSegmentBase *pSegment, ScheduleGroupSegmentBase *pSkipSegment, SearchAffinity affinity, bool fLastPass); /// /// Searches the schedule group to which pSegment belongs to find a runnable. The group is searched for segments according to the specified /// affinity type. /// /// /// If an appropriate runnable is found, the resulting work item will be placed here. /// /// /// A segment within the group to search. This segment has bias within the group if it matches the specified affinity type. /// /// /// An indication as to whether this is a last pass SFW. /// /// /// An indication of whether a work item was found or not. /// bool GetRunnableContextWithinGroup(WorkItem *pWorkItem, ScheduleGroupSegmentBase *pSegment, SearchAffinity affinity, bool fLastPass); /// /// Searches the schedule group to which pSegment belongs to find a realized chore. The group is searched for segments according to the specified /// affinity type. /// /// /// If an appropriate realized chore is found, the resulting work item will be placed here. /// /// /// A segment within the group to search. This segment has bias within the group if it matches the specified affinity type. /// /// /// An indication as to whether this is a last pass SFW. /// /// /// An indication of whether a work item was found or not. /// bool GetRealizedChoreWithinGroup(WorkItem *pWorkItem, ScheduleGroupSegmentBase *pSegment, bool fRealWork, SearchAffinity affinity, bool fLastPass); /// /// Searches the schedule group to which pSegment belongs to find an unrealized chore. The group is searched for segments according to the /// specified affinity type. /// /// /// If an appropriate unrealized chore is found, the resulting work item will be placed here. /// /// /// A segment within the group to search. This segment has bias within the group if it matches the specified affinity type. /// /// /// An indication as to whether this is a last pass SFW. /// /// /// An indication of whether a work item was found or not. /// bool GetUnrealizedChoreWithinGroup(WorkItem *pWorkItem, ScheduleGroupSegmentBase *pSegment, bool fRealWork, SearchAffinity affinity, bool fLastPass); /// /// Called on any biased work. /// void LRCBias() { m_LRCBias++; } /// /// Resets the local bias counter but not the ring bias counter. /// void ResetLRCBias() { m_LRCBias = 0; } /// /// Returns the current stage of local bias. /// BiasStageType BiasStage() { if (m_LRCBias < 101) return BiasStageNone; // (fwd) Normal --> LRC LIFO else if (m_LRCBias < 127) return BiasStageFlipLRC; // (fwd) Flip LRC --> LRC FIFO else return BiasStageSkipLRC; // (fwd) Skip LRC --> runnables } // ************************************************** // Cache Local Algorithm: // /// /// Searches for a runnable within the specified ring. Before searching elsewhere, it searches the segment and group specified by /// pBiasSegment according to the rules of the search and the requested affinity type. /// /// /// If a work item is found, the work item will be returned here. /// /// /// The scheduling ring to search. /// /// /// The segment to bias the search to. This segment and its corresponding group are searched first! /// /// /// Determines whether or not to check other local LRCs in this search. /// /// /// The search affinity type to query for. /// /// /// A bitmap of work-types allowed to be returned from the search. This indicates whether or not runnables, realized chores, or unrealized chores /// can be returned as well as whether the actual work item or only a token should be returned. /// /// /// An indication as to whether this is a last pass SFW. /// /// /// An indication of whether a runnable was found in the bias segment, group, or the specified ring. /// bool SearchCacheLocal_Runnables(WorkItem *pWorkItem, SchedulingRing *pRing, ScheduleGroupSegmentBase *pBiasSegment, bool fOtherLocalLRCCheck, SearchAffinity affinity, ULONG allowableTypes, bool fLastPass); /// /// Searches for a realized chore within the specified ring. Before searching elsewhere, it searches the segment and group specified by /// pBiasSegment according to the rules of the search and the requested affinity type. /// /// /// If a work item is found, the work item will be returned here. /// /// /// The scheduling ring to search. /// /// /// The segment to bias the search to. This segment and its corresponding group are searched first! /// /// /// If true, the actual work item is returned. If false, a token to the work is returned. The work is not dequeued until the token /// is resolved. /// /// /// The search affinity type to query for. /// /// /// A bitmap of work-types allowed to be returned from the search. This indicates whether or not runnables, realized chores, or unrealized chores /// can be returned as well as whether the actual work item or only a token should be returned. /// /// /// An indication as to whether this is a last pass SFW. /// /// /// An indication of whether a realized chore was found in the bias segment, group, or the specified ring. /// bool SearchCacheLocal_Realized(WorkItem *pWorkItem, SchedulingRing *pRing, ScheduleGroupSegmentBase *pBiasSegment, bool fRealWork, SearchAffinity affinity, ULONG allowableTypes, bool fLastPass); /// /// Searches for an unrealized chore within the specified ring. Before searching elsewhere, it searches the segment and group specified by /// pBiasSegment according to the rules of the search and the requested affinity type. /// /// /// If a work item is found, the work item will be returned here. /// /// /// The scheduling ring to search. /// /// /// The segment to bias the search to. This segment and its corresponding group are searched first! /// /// /// If true, the actual work item is returned. If false, a token to the work is returned. The work is not dequeued until the token /// is resolved. /// /// /// The search affinity type to query for. /// /// /// A bitmap of work-types allowed to be returned from the search. This indicates whether or not runnables, realized chores, or unrealized chores /// can be returned as well as whether the actual work item or only a token should be returned. /// /// /// An indication as to whether this is a last pass SFW. /// /// /// An indication of whether an unrealized chore was found in the bias segment, group, or the specified ring. /// bool SearchCacheLocal_Unrealized(WorkItem *pWorkItem, SchedulingRing *pRing, ScheduleGroupSegmentBase *pBiasSegment, bool fRealWork, SearchAffinity affinity, ULONG allowableTypes, bool fLastPass); /// /// Searches for work within the scheduler according to the cache local (schedule group local) search algorithm. /// /// /// If a work item is found, the work item will be returned here. /// /// /// The segment to bias the search to. /// /// /// An indication as to whether this is a last pass SFW. /// /// /// A bitmap of work-types allowed to be returned from the search. This indicates whether or not runnables, realized chores, or unrealized chores /// can be returned as well as whether the actual work item or only a token should be returned. /// /// /// An indication of whether a work item was found or not. /// bool SearchCacheLocal(WorkItem *pWorkItem, ScheduleGroupSegmentBase *pOriginSegment, bool fLastPass, ULONG allowableTypes); /// /// Searches for work within the scheduler according to the cache local (schedule group local) search algorithm for yielding. /// /// /// If a work item is found, the work item will be returned here. /// /// /// The segment to bias the search to. /// /// /// An indication as to whether this is a last pass SFW. /// /// /// A bitmap of work-types allowed to be returned from the search. This indicates whether or not runnables, realized chores, or unrealized chores /// can be returned as well as whether the actual work item or only a token should be returned. /// /// /// An indication of whether a work item was found or not. /// bool SearchCacheLocalYield(WorkItem *pWorkItem, ScheduleGroupSegmentBase *pOriginSegment, bool fLastPass, ULONG allowableTypes); // ************************************************** // Fair Algorithm: // /// /// Performs a fair search for runnables in the specified ring. /// bool SearchFair_Runnables(WorkItem *pWorkItem, SchedulingRing *pRing); /// /// Performs a fair search for realized chores in the specified ring. /// bool SearchFair_Realized(WorkItem *pWorkItem, SchedulingRing *pRing, bool fRealItem); /// /// Performs a fair search for unrealized chores in the specified ring. /// bool SearchFair_Unrealized(WorkItem *pWorkItem, SchedulingRing *pRing, bool fRealItem); /// /// Performs a fair search for work. /// bool SearchFair(WorkItem *pWorkItem, ScheduleGroupSegmentBase *pOriginSegment, bool fLastPass, ULONG allowableTypes); /// /// Performs a fair search for work in the yielding case. /// bool SearchFairYield(WorkItem *pWorkItem, ScheduleGroupSegmentBase *pOriginSegment, bool fLastPass, ULONG allowableTypes); }; } }