deep_scan.py

import torch
from transformers import AutoConfig, AutoModelForCausalLM
from colorama import Fore, Style, init

init()
print(Fore.CYAN + "--- TFC DEEP PARAMETER SCAN ---" + Style.RESET_ALL)

MODEL_ID = "Qwen/Qwen1.5-MoE-A2.7B"

# 1. Load Skeleton
print("Loading Skeleton...")
config = AutoConfig.from_pretrained(MODEL_ID, trust_remote_code=True)
model = AutoModelForCausalLM.from_config(config, trust_remote_code=True)

# 2. Target the Experts Container
print(Fore.YELLOW + "\nSCANNING EXPERT PARAMETERS (Layer 0)..." + Style.RESET_ALL)
experts_container = model.model.layers[0].mlp.experts

# 3. List all parameters directly
params = list(experts_container.named_parameters())

if len(params) == 0:
    print(Fore.RED + "CRITICAL: No parameters found in 'experts' container!" + Style.RESET_ALL)
    print("Checking parent MLP...")
    for name, _ in model.model.layers[0].mlp.named_parameters():
        print(f"   (MLP Level) Found: {name}")
else:
    for name, param in params:
        print(f"   Name: '{Fore.GREEN}{name}{Style.RESET_ALL}'")
        print(f"     -> Shape: {param.shape}")
        print(f"     -> Type:  {type(param)}")

print(Fore.CYAN + "\n--- CONCLUSION ---" + Style.RESET_ALL)
if any("gate_up" in n for n, _ in params):
    print("We found 'gate_up'. The previous script failed because it looked for .weight")
elif any("weight" in n for n, _ in params):
    print("We found standard weights. The paths might be different.")

print("Done.")