Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 9 additions & 4 deletions c/qwen36_tier.c
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
#include <pthread.h>
#include "qwen36_tier.h"
#include "backend_cuda.h"
#include "tier.h"

#define QT_MAX_DEV 8
#define QT_QCAP 48 /* upload queue depth (staging ~1.6 MB/entry) */
Expand Down Expand Up @@ -330,11 +331,15 @@ void qt_fill_wait(void){
pthread_mutex_unlock(&G.mx);
}

/* LFRU swap check (every 16 ticks = tokens): per device, coldest resident vs
* hottest non-resident, with the tier.h hysteresis. */
/* Adaptive swap check (every 16 ticks = tokens): per device, coldest resident
* vs hottest non-resident. Decay every 1024 ticks so an old workload cannot
* permanently own the tier; admission uses the shared tier.h contract. */
static void qt_lfru_tick_locked(void){
if(++G.tick % 16) return;
size_t n=(size_t)G.nl*G.ne;
G.tick++;
if(!(G.tick%1024))
for(size_t i=0;i<n;i++) G.slot[i].heat=tier_decay_value(G.slot[i].heat);
if(G.tick%16) return;
for(int di=0;di<G.ndev;di++){
int cold=-1, hot=-1; uint32_t ch=0, hh=0;
for(size_t i=0;i<n;i++){
Expand All @@ -345,7 +350,7 @@ static void qt_lfru_tick_locked(void){
else if(!s->resident && !s->queued && s->g4){ if(hot<0||s->heat>hh){ hot=(int)i; hh=s->heat; } }
}
if(cold<0||hot<0) continue;
if(hh<=ch+(ch>>2)+4) continue; /* hysteresis as in tier.h */
if(!tier_should_promote(hh,ch)) continue;
QSlot *v=&G.slot[cold];
v->resident=0; /* CPU fallback from now on */
if(enqueue_locked(hot/G.ne,hot%G.ne,cold/G.ne,cold%G.ne,0)) G.swaps++;
Expand Down
5 changes: 5 additions & 0 deletions c/tests/test_tier.c
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,11 @@ static int fail(const char *message){
}

int main(void){
if(!tier_should_promote(130,100)) return fail("shared hysteresis admits hot expert");
if(tier_should_promote(129,100)) return fail("shared hysteresis blocks marginal expert");
if(tier_should_promote(UINT32_MAX-1,UINT32_MAX))
return fail("saturated hysteresis threshold wrapped");
if(tier_decay_value(9)!=4) return fail("shared heat decay");
uint32_t heat[6]={20,2,8,3,30,1};
int pinned[2]={0,1}, slot=-1, eid=-1; long gain=0;
if(!tier_pick_swap(heat,6,pinned,2,&slot,&eid,&gain)) return fail("hot expert not promoted");
Expand Down
14 changes: 12 additions & 2 deletions c/tier.h
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,16 @@

#include <stdint.h>

/* Shared admission contract for every adaptive resident tier. Widen before
* adding the margin: a saturated uint32 heat counter must become sticky, not
* wrap the threshold and admit a colder expert. */
static int tier_should_promote(uint32_t hot, uint32_t cold){
uint64_t threshold=(uint64_t)cold+((uint64_t)cold>>2)+4u;
return (uint64_t)hot>threshold;
}

static uint32_t tier_decay_value(uint32_t heat){ return heat>>1; }

/* Pick one RAM/VRAM hot-store slot to replace from recent routing heat.
* The fixed margin handles tiny samples; the 25% margin prevents ping-pong. */
static int tier_pick_swap(const uint32_t *heat, int nexpert,
Expand All @@ -19,7 +29,7 @@ static int tier_pick_swap(const uint32_t *heat, int nexpert,
}
if(hot<0) return 0;
uint32_t fc=heat[pinned[cold]];
if(fh<=fc+(fc>>2)+4) return 0;
if(!tier_should_promote(fh,fc)) return 0;
*slot=cold; *eid=hot; *gain=(long)fh-(long)fc;
return 1;
}
Expand Down Expand Up @@ -54,7 +64,7 @@ static int tier_pick_lfru(const uint32_t *heat, const uint32_t *last, uint32_t c
}

static void tier_decay(uint32_t *heat, int nexpert){
for(int e=0;e<nexpert;e++) heat[e]>>=1;
for(int e=0;e<nexpert;e++) heat[e]=tier_decay_value(heat[e]);
}

#endif
4 changes: 3 additions & 1 deletion docs/qwen36-cuda-tier.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,9 @@ more GPUs and computed there through the existing shared CUDA backend

- **Home device:** expert `eid` lives on GPU `eid % n_gpus`; no duplicates.
- **Placement:** routing heat decides who earns VRAM (LFRU semantics from
`tier.h`, 25%+4 hysteresis). A parallel **warmstart** fills the per-device
`tier.h`, 25%+4 hysteresis). Runtime heat is halved every 1024 decode ticks,
so a long-lived process can replace experts from an old workload instead of
permanently freezing its initial hot set. A parallel **warmstart** fills the per-device
budget before the first token — ordered by a persisted heat table
(`HEAT_FILE`) when present, so a second run starts fully placed.
- **Decode:** per (token, layer) the resident experts are issued as async
Expand Down
Loading