Wire breakdown.csv into feedsim_autoscale_dlrm + add preprocessing/postprocessing logs (#736)

benchpress/config/jobs.yml

@@ -646,6 +646,10 @@
     - '--client-feature-seed={client_feature_seed}'
     - '--client-num-dense={client_num_dense}'
     - '--client-num-sparse={client_num_sparse}'
+    - '--feature-extractors={feature_extractors}'
+    - '--feature-complexity={feature_complexity}'
+    - '--num-stories={num_stories}'
+    - '--extractors-per-story={extractors_per_story}'
   vars:
     - 'port=11222'
     - 'output=feedsim_results.txt'
@@ -662,6 +666,10 @@
     - 'client_feature_seed=42'
     - 'client_num_dense=13'
     - 'client_num_sparse=26'
+    - 'feature_extractors=0'
+    - 'feature_complexity=5'
+    - 'num_stories=100'
+    - 'extractors_per_story=50'
   hooks:
     - hook: cpu-mpstat
       options:
@@ -731,6 +739,12 @@
     - '--client-feature-seed={client_feature_seed}'
     - '--client-num-dense={client_num_dense}'
     - '--client-num-sparse={client_num_sparse}'
+    - '--feature-extractors={feature_extractors}'
+    - '--feature-complexity={feature_complexity}'
+    - '--num-stories={num_stories}'
+    - '--extractors-per-story={extractors_per_story}'
+    - '--silesia-dir={silesia_dir}'
+    - '--stories-per-request={stories_per_request}'
     - '{extra_args}'
   vars:
     - 'num_instances=-1'
@@ -747,6 +761,12 @@
     - 'client_feature_seed=42'
     - 'client_num_dense=13'
     - 'client_num_sparse=26'
+    - 'feature_extractors=0'
+    - 'feature_complexity=5'
+    - 'num_stories=100'
+    - 'extractors_per_story=50'
+    - 'silesia_dir=silesia'
+    - 'stories_per_request=10'
     - 'extra_args='
   hooks:
     - hook: cpu-mpstat
@@ -761,6 +781,7 @@
           - 'benchmarks/feedsim/feedsim_results*.txt'
           - 'benchmarks/feedsim/feedsim-multi-inst-*.log'
           - 'benchmarks/feedsim/src/perf.data'
+          - 'benchmarks/feedsim/breakdown.csv'
           - '/tmp/feedsim_log.txt'
 
 - name: feedsim_autoscale_dlrm_mini
@@ -785,6 +806,10 @@
     - '--client-feature-seed={client_feature_seed}'
     - '--client-num-dense={client_num_dense}'
     - '--client-num-sparse={client_num_sparse}'
+    - '--feature-extractors'
+    - '--feature-complexity={feature_complexity}'
+    - '--num-stories={num_stories}'
+    - '--extractors-per-story={extractors_per_story}'
     - '-q {fixed_qps}'
     - '-d {fixed_qps_duration}'
     - '-w {warmup_time}'
@@ -804,13 +829,21 @@
     - 'dlrm_model=models/dlrm_small.pt'
     - 'dlrm_batch_size=256'
     - 'dlrm_threads=1'
-    - 'dlrm_inferences=64'
+    # 8 inferences/req brings DLRM-Inference CPU share from 55-62% down
+    # toward prod multifeed_aggregator's Ranking-Prediction share of 7-13%.
+    - 'dlrm_inferences=8'
     - 'client_side_features=0'
     - 'client_batch_size=256'
     - 'client_inferences=64'
     - 'client_feature_seed=42'
     - 'client_num_dense=13'
     - 'client_num_sparse=26'
+    - 'feature_complexity=5'
+    # 1800 stories x 50 extractors = 90K extractor calls/req. Tuned to
+    # push FeatureExtraction CPU share into prod's 30-35% band; 5K
+    # calls/req previously gave only 1.7-1.9%.
+    - 'num_stories=1800'
+    - 'extractors_per_story=50'
     - 'fixed_qps=100000'
     - 'fixed_qps_duration=10'
     - 'warmup_time=5'

packages/common/runtime_breakdown_utils.sh

@@ -25,6 +25,16 @@ create_breakdown_csv() {
 
     local csv_file="${folder_path}/${breakdown_file_name}"
 
+    # Idempotent: if the file already exists, leave it alone. Multi-instance
+    # workloads (e.g. feedsim run-feedsim-multi.sh) have several processes
+    # racing to call create_breakdown_csv concurrently — truncating after the
+    # first writer would silently drop entries already logged by the earlier
+    # instance. benchpress's copymove hook (is_move: true) clears the file
+    # between iterations, so stale data is not a risk.
+    if [ -f "$csv_file" ]; then
+        return 0
+    fi
+
     # Create CSV file with headers
     if echo "operation_name,PID,timestamp_type,timestamp,sub_operation_name" > "$csv_file"; then
         echo "Created breakdown CSV file: $csv_file"

packages/feedsim/feed_aggregator_req_sizes.json

@@ -0,0 +1 @@
+[{"req_size_min":9,"req_size_p05":55,"req_size_p10":56,"req_size_p15":87,"req_size_p20":353,"req_size_p25":354,"req_size_p30":361,"req_size_p35":368,"req_size_p40":35168,"req_size_p45":42544,"req_size_p50":47874,"req_size_p55":55587,"req_size_p60":388656,"req_size_p65":1084512,"req_size_p70":1522044,"req_size_p75":1921331,"req_size_p80":2239067,"req_size_p85":2584693,"req_size_p90":3025469,"req_size_p95":3679075,"req_size_p99":4075462,"req_size_max":18882521}]

packages/feedsim/feed_aggregator_resp_sizes.json

@@ -0,0 +1 @@
+[{"resp_size_min":4,"resp_size_p05":4,"resp_size_p10":4,"resp_size_p15":4,"resp_size_p20":4,"resp_size_p25":4,"resp_size_p30":4,"resp_size_p35":4,"resp_size_p40":4,"resp_size_p45":42,"resp_size_p50":44,"resp_size_p55":44,"resp_size_p60":46,"resp_size_p65":30473,"resp_size_p70":128474,"resp_size_p75":148553,"resp_size_p80":217715,"resp_size_p85":873710,"resp_size_p90":1303070,"resp_size_p95":2100241,"resp_size_p99":2594795,"resp_size_max":10319524}]

packages/feedsim/generate_dlrm_models.py

@@ -0,0 +1,171 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""Generate DLRM medium and large TorchScript models with random weights.
+
+For benchmarking, trained weights aren't needed — the compute pattern
+(embedding lookups, MLP forward passes, feature interactions) is identical
+regardless of weight values. These models are architecture-compatible with
+the existing dlrm_small.pt model used by FeedSim's DLRM inference path.
+
+Usage:
+    python3 generate_dlrm_models.py <output_dir>
+"""
+
+import os
+import sys
+
+import torch
+from torch import nn
+
+
+class DLRM(nn.Module):
+    """Simplified DLRM model matching production inference patterns."""
+
+    def __init__(
+        self,
+        emb_dim: int = 64,
+        num_dense: int = 13,
+        num_sparse: int = 26,
+        max_emb_rows: int = 250000,
+        bottom_mlp_dims: list = None,
+        top_mlp_dims: list = None,
+    ):
+        super().__init__()
+        if bottom_mlp_dims is None:
+            bottom_mlp_dims = [256, 128, emb_dim]
+        if top_mlp_dims is None:
+            top_mlp_dims = [256, 128, 1]
+
+        # Bottom MLP: dense features -> embedding dimension
+        layers = []
+        in_dim = num_dense
+        for out_dim in bottom_mlp_dims:
+            layers.append(nn.Linear(in_dim, out_dim))
+            layers.append(nn.ReLU())
+            in_dim = out_dim
+        self.bottom_mlp = nn.Sequential(*layers)
+
+        # Embedding tables for sparse features
+        emb_sizes = [
+            40000000,
+            39060,
+            17295,
+            7424,
+            20265,
+            3,
+            7122,
+            1543,
+            63,
+            40000000,
+            3067956,
+            405282,
+            10,
+            2209,
+            11938,
+            155,
+            4,
+            976,
+            14,
+            40000000,
+            40000000,
+            40000000,
+            590152,
+            12973,
+            108,
+            36,
+        ]
+        # Use min(actual_size, max_emb_rows) to control model size
+        self.embeddings = nn.ModuleList(
+            [
+                nn.EmbeddingBag(min(s, max_emb_rows), emb_dim, mode="sum")
+                for s in emb_sizes[:num_sparse]
+            ]
+        )
+
+        # Top MLP: interaction output -> prediction. ReLU only on hidden
+        # layers; the final Linear feeds raw logits into sigmoid in forward().
+        n = 1 + num_sparse  # bottom_mlp output + embedding outputs
+        interaction_size = emb_dim + (n * (n - 1)) // 2
+        top_layers = []
+        in_dim = interaction_size
+        for i, out_dim in enumerate(top_mlp_dims):
+            top_layers.append(nn.Linear(in_dim, out_dim))
+            if i < len(top_mlp_dims) - 1:
+                top_layers.append(nn.ReLU())
+            in_dim = out_dim
+        self.top_mlp = nn.Sequential(*top_layers)
+
+    def forward(self, dense: torch.Tensor, sparse: torch.Tensor) -> torch.Tensor:
+        # Bottom MLP
+        d = self.bottom_mlp(dense)
+
+        # Embedding lookups
+        embs = [emb(sparse[:, i].unsqueeze(1)) for i, emb in enumerate(self.embeddings)]
+
+        # Feature interaction (dot product)
+        combined = torch.cat([d.unsqueeze(1)] + [e.unsqueeze(1) for e in embs], dim=1)
+        interact = torch.bmm(combined, combined.transpose(1, 2))
+        n = combined.size(1)
+        idx = torch.triu_indices(n, n, offset=1)
+        flat = interact[:, idx[0], idx[1]]
+
+        # Top MLP
+        x = torch.cat([d, flat], dim=1)
+        return torch.sigmoid(self.top_mlp(x))
+
+
+def generate_model(output_path: str, **kwargs):
+    """Generate and save a TorchScript DLRM model."""
+    model = DLRM(**kwargs)
+    param_bytes = sum(p.numel() * p.element_size() for p in model.parameters())
+    print(f"  Parameters: {sum(p.numel() for p in model.parameters()):,}")
+    print(f"  Model size: {param_bytes / 1e6:.0f} MB")
+
+    scripted = torch.jit.script(model)
+    scripted.save(output_path)
+    file_size = os.path.getsize(output_path)
+    print(f"  Saved to: {output_path} ({file_size / 1e6:.0f} MB on disk)")
+
+
+def main():
+    if len(sys.argv) < 2:
+        print(f"Usage: {sys.argv[0]} <output_dir>")
+        sys.exit(1)
+
+    output_dir = sys.argv[1]
+    os.makedirs(output_dir, exist_ok=True)
+
+    # Medium model: larger embeddings (~500MB)
+    medium_path = os.path.join(output_dir, "dlrm_medium.pt")
+    if not os.path.exists(medium_path):
+        print("Generating DLRM medium model...")
+        generate_model(
+            medium_path,
+            emb_dim=64,
+            max_emb_rows=500000,
+            bottom_mlp_dims=[256, 128, 64],
+            top_mlp_dims=[256, 128, 1],
+        )
+    else:
+        print(f"[SKIPPED] {medium_path} already exists")
+
+    # Large model: even larger embeddings (~1GB)
+    large_path = os.path.join(output_dir, "dlrm_large.pt")
+    if not os.path.exists(large_path):
+        print("Generating DLRM large model...")
+        generate_model(
+            large_path,
+            emb_dim=128,
+            max_emb_rows=500000,
+            bottom_mlp_dims=[512, 256, 128],
+            top_mlp_dims=[512, 256, 1],
+        )
+    else:
+        print(f"[SKIPPED] {large_path} already exists")
+
+
+if __name__ == "__main__":
+    main()

packages/feedsim/install_feedsim.sh

@@ -58,6 +58,12 @@ cp "${BENCHPRESS_ROOT}/packages/feedsim/run-feedsim-multi.sh" "${FEEDSIM_ROOT_SR
 chmod u+x "${FEEDSIM_ROOT_SRC}/run.sh"
 chmod u+x "${FEEDSIM_ROOT_SRC}/run-feedsim-multi.sh"
 
+# Copy production size distribution JSONs (consumed by run.sh / DriverNodeRank).
+cp "${BENCHPRESS_ROOT}/packages/feedsim/feed_aggregator_req_sizes.json" "${FEEDSIM_ROOT_SRC}/feed_aggregator_req_sizes.json"
+cp "${BENCHPRESS_ROOT}/packages/feedsim/feed_aggregator_resp_sizes.json" "${FEEDSIM_ROOT_SRC}/feed_aggregator_resp_sizes.json"
+# Phase 6 session-mode driver loads rpc_dist.json from the FEEDSIM_ROOT runtime dir.
+cp "${BENCHPRESS_ROOT}/packages/feedsim/rpc_dist.json" "${FEEDSIM_ROOT_SRC}/rpc_dist.json"
+
 msg "Installing third-party dependencies..."
 cp -r "${BENCHPRESS_ROOT}/packages/feedsim/third_party" "${FEEDSIM_ROOT_SRC}"
 mv "${FEEDSIM_THIRD_PARTY_SRC}/src" "${FEEDSIM_ROOT_SRC}/src"
@@ -214,6 +220,32 @@ else
 fi
 
 
+# Download Silesia compression corpus for story-based requests (Phase 3)
+SILESIA_DIR="${FEEDSIM_ROOT_SRC}/silesia"
+SILESIA_URL="https://github.com/facebookresearch/DCPerf-datasets/releases/download/feedsim-silesia/silesia.tar.gz"
+if ! [ -d "$SILESIA_DIR" ] || [ -z "$(ls -A "$SILESIA_DIR" 2>/dev/null)" ]; then
+    msg "Downloading Silesia corpus..."
+    mkdir -p "$SILESIA_DIR"
+    cd "$SILESIA_DIR" || { msg "[ERROR] cannot cd to $SILESIA_DIR"; exit 1; }
+    if wget -q "$SILESIA_URL" -O silesia.tar.gz 2>/dev/null; then
+        tar -xzf silesia.tar.gz && rm -f silesia.tar.gz
+        msg "Silesia corpus downloaded: $(ls | wc -l) files, $(du -sh . | cut -f1)"
+    else
+        # Fallback to original Silesia host
+        msg "[INFO] GitHub dataset not available, trying original Silesia host..."
+        SILESIA_FALLBACK_URL="https://sun.aei.polsl.pl/~sdeor/corpus/silesia.zip"
+        if wget -q "$SILESIA_FALLBACK_URL" -O silesia.zip 2>/dev/null; then
+            unzip -q silesia.zip && rm -f silesia.zip
+            msg "Silesia corpus downloaded: $(ls | wc -l) files, $(du -sh . | cut -f1)"
+        else
+            msg "[WARNING] Silesia download failed — story-based requests will be unavailable"
+        fi
+    fi
+    cd "${FEEDSIM_THIRD_PARTY_SRC}"
+else
+    msg "[SKIPPED] Silesia corpus already present at $SILESIA_DIR"
+fi
+
 # Installing FeedSim
 cd "${FEEDSIM_ROOT_SRC}"
 
@@ -242,6 +274,16 @@ if [ -f "third_party/fizz/fizz/tool/FizzServerCommand.cpp" ]; then
     sed -i 's/EVP_PKEY_cmp(pubKey.get(), key.get()) == 1/EVP_PKEY_eq(pubKey.get(), key.get())/g' "third_party/fizz/fizz/tool/FizzServerCommand.cpp"
 fi
 
+# Generate feature extractor variants (1M+ unique functions for I-cache pressure)
+msg "Generating feature extractor variants..."
+CODEGEN_DIR="${FEEDSIM_ROOT_SRC}/src/workloads/ranking/feature_extractors/generated"
+if [ -f "${CODEGEN_DIR}/generate_extractors.py" ]; then
+    python3 "${CODEGEN_DIR}/generate_extractors.py" --output-dir "${CODEGEN_DIR}"
+    msg "Feature extractor codegen complete"
+else
+    msg "[SKIPPED] No codegen script found at ${CODEGEN_DIR}/generate_extractors.py"
+fi
+
 mkdir -p build && cd build/
 
 # Build FeedSim with DLRM support
@@ -261,7 +303,14 @@ cmake -G Ninja \
     -DCMAKE_PREFIX_PATH="${FEEDSIM_THIRD_PARTY_SRC}/libtorch" \
     ../
 
-ninja -v -j1
+# Dependencies (fmt, folly, fbthrift, etc.) are already installed above via
+# separate make commands in their own build directories. This ninja step only
+# builds FeedSim itself (LeafNodeRank, DriverNodeRank, feature extractors),
+# so parallel builds are safe here. Use nproc/2 to avoid OOM.
+NINJA_JOBS="${BP_NINJA_JOBS:-$(( $(nproc) / 2 ))}"
+[ "$NINJA_JOBS" -lt 1 ] && NINJA_JOBS=1
+msg "Building FeedSim with ninja -j${NINJA_JOBS} (set BP_NINJA_JOBS to override)"
+ninja -j"${NINJA_JOBS}"
 
 msg ""
 msg "=== FeedSim Installation Complete ==="