{
 "campaign": "mlx-20261103-234700",
 "hardware": {
  "mini": {
   "chip": "Apple M6",
   "physical_memory_gib": 16
  },
  "studio": {
   "chip": "Apple M5 Max",
   "physical_memory_gib": 64
  }
 },
 "comparison": "same absolute model allowance, not physical-RAM emulation",
 "model_budget_bytes": 11441804083,
 "models": [
  {
   "repo": "mlx-community/Qwen3.5-4B-MLX-4bit",
   "revision": "32f3e8ecf65426fc3306969496342d504bfa13f3",
   "weights_bytes": 3034300695,
   "model_type": "qwen3_5",
   "missing_shards": [],
   "config_quantization": {
    "group_size": 64,
    "bits": 4,
    "mode": "affine"
   },
   "upstream": "Qwen/Qwen3.5-4B"
  },
  {
   "repo": "mlx-community/Qwen3.5-9B-MLX-4bit",
   "revision": "938d8919941c6e7efd3c7150eff7fe9d12afa631",
   "weights_bytes": 5950221072,
   "model_type": "qwen3_5",
   "missing_shards": [],
   "config_quantization": {
    "group_size": 64,
    "bits": 4,
    "mode": "affine"
   },
   "upstream": "Qwen/Qwen3.5-9B"
  },
  {
   "repo": "mlx-community/gemma-4-12B-it-4bit",
   "revision": "73bcf09092aa277861d5a191b989b666f7f32e8f",
   "weights_bytes": 6741039511,
   "model_type": "gemma4_unified",
   "missing_shards": [],
   "config_quantization": {
    "group_size": 64,
    "bits": 4,
    "mode": "affine"
   },
   "upstream": null
  },
  {
   "repo": "mlx-community/LFM2.5-8B-A1B-MLX-4bit",
   "revision": "146590a491db88581884033023f51f6b49a27b89",
   "weights_bytes": 4764265165,
   "model_type": "lfm2_moe",
   "missing_shards": [],
   "config_quantization": {
    "group_size": 64,
    "bits": 4,
    "mode": "affine",
    "model.layers.2.feed_forward.gate": {
     "group_size": 64,
     "bits": 8
    },
    "model.layers.3.feed_forward.gate": {
     "group_size": 64,
     "bits": 8
    },
    "model.layers.4.feed_forward.gate": {
     "group_size": 64,
     "bits": 8
    },
    "model.layers.5.feed_forward.gate": {
     "group_size": 64,
     "bits": 8
    },
    "model.layers.6.feed_forward.gate": {
     "group_size": 64,
     "bits": 8
    },
    "model.layers.7.feed_forward.gate": {
     "group_size": 64,
     "bits": 8
    },
    "model.layers.8.feed_forward.gate": {
     "group_size": 64,
     "bits": 8
    },
    "model.layers.9.feed_forward.gate": {
     "group_size": 64,
     "bits": 8
    },
    "model.layers.10.feed_forward.gate": {
     "group_size": 64,
     "bits": 8
    },
    "model.layers.11.feed_forward.gate": {
     "group_size": 64,
     "bits": 8
    },
    "model.layers.12.feed_forward.gate": {
     "group_size": 64,
     "bits": 8
    },
    "model.layers.13.feed_forward.gate": {
     "group_size": 64,
     "bits": 8
    },
    "model.layers.14.feed_forward.gate": {
     "group_size": 64,
     "bits": 8
    },
    "model.layers.15.feed_forward.gate": {
     "group_size": 64,
     "bits": 8
    },
    "model.layers.16.feed_forward.gate": {
     "group_size": 64,
     "bits": 8
    },
    "model.layers.17.feed_forward.gate": {
     "group_size": 64,
     "bits": 8
    },
    "model.layers.18.feed_forward.gate": {
     "group_size": 64,
     "bits": 8
    },
    "model.layers.19.feed_forward.gate": {
     "group_size": 64,
     "bits": 8
    },
    "model.layers.20.feed_forward.gate": {
     "group_size": 64,
     "bits": 8
    },
    "model.layers.21.feed_forward.gate": {
     "group_size": 64,
     "bits": 8
    },
    "model.layers.22.feed_forward.gate": {
     "group_size": 64,
     "bits": 8
    },
    "model.layers.23.feed_forward.gate": {
     "group_size": 64,
     "bits": 8
    }
   },
   "upstream": "LiquidAI/LFM2.5-8B-A1B"
  },
  {
   "repo": "mlx-community/Phi-4-mini-instruct-mlx-4Bit",
   "revision": "d848c30f6d5419b9892433cf6b1062626d15340e",
   "weights_bytes": 2158101119,
   "model_type": "phi3",
   "config_quantization": {
    "group_size": 64,
    "bits": 4,
    "mode": "affine"
   },
   "upstream": "microsoft/Phi-4-mini-instruct"
  }
 ],
 "runtime_versions": {
  "mlx-lm": "0.32.2",
  "mlx": "0.32.4",
  "pi": "1.5.0"
 },
 "core_settings": {
  "temperature": 0,
  "seed": 42,
  "context": 32768,
  "practical_max_output_tokens": 512,
  "pi_max_output_tokens": 2048,
  "pi_task_timeout_seconds": 300,
  "prefix_cache_entries": 1,
  "prefix_cache_bytes": "256MB",
  "thinking": "disabled where chat template supports it"
 },
 "workflow_settings": {
  "pi_max_output_tokens": 8192,
  "task_timeout_seconds": 300,
  "model_timeout_seconds": 3300
 },
 "video_settings": {
  "phase": "video-edit-pi-v1",
  "created_epoch": 1793797928.967621,
  "models": "same five pinned MLX models as core",
  "harness": "stock Pi 1.5.0",
  "output_token_cap": 8192,
  "context": 32768,
  "temperature": 0,
  "seed": 42,
  "tasks": 5,
  "task_timeout_seconds": 300,
  "model_timeout_seconds": 1680,
  "common_phase_seconds": 9000,
  "budget_bytes": 11441804083,
  "runtime": "imageio-ffmpeg0.8.1 bundled FFmpeg9.0 identical binary hash both hosts",
  "scope": "LLM-orchestrated deterministic editing of synthetic clips; NOT video perception or video generation",
  "timing": "Pi start-to-exit wall time; validator and staging excluded. FFmpeg work included. Server native timing separate.",
  "memory_limit_scope": "Existing MLX server footprint/allocator/swap guard; FFmpeg child footprint not individually capped; low resolution tiny fixture bounds processing. Not full-machine physical RAM emulation.",
  "sources": [
   "https://github.com/imageio/imageio-ffmpeg"
  ],
  "no_bad_answer_retries": true
 },
 "audio_settings": {
  "protocol": "audio-v1",
  "models": [
   "whisper-small-mlx",
   "whisper-large-v3-turbo"
  ],
  "fixtures": 22,
  "repeats": 3,
  "runtime": "mlx-whisper==0.4.3; mlx==0.32.4",
  "memory_bytes": 11441804083,
  "temperature": 0,
  "language": "en",
  "fp16": true,
  "condition_on_previous_text": true,
  "initial_prompt": null,
  "normalization": "NFKC lower-case lexical WER, internal apostrophes preserved, no number or contraction expansion",
  "metrics": [
   "lexical_wer",
   "wall_seconds",
   "real_time_factor",
   "silence_hallucination",
   "observed_memory",
   "swap_growth"
  ],
  "reserve_final_analysis_seconds": 3600,
  "limitations": "Tiny one-speaker read-speech subset plus synthetic practical speech, artificial white noise and silence. Not a population accuracy or real-meeting estimate. Equivalent number formatting may count as lexical errors."
 },
 "image_settings": {
  "phase": "image-v1",
  "model": "stabilityai/sdxl-turbo",
  "revision": "71153311d3dbb46851df1931d3ca6e939de83304",
  "runtime": "official ml-explore/mlx-examples stable_diffusion",
  "code_commit": "042280ce50645b69d9c322ccf1cb8471384007f1",
  "width": 512,
  "height": 512,
  "batch_size": 1,
  "unet_bits": 8,
  "unet_group_size": 32,
  "text_encoder_bits": 4,
  "text_encoder_group_size": 64,
  "vae_dtype": "float32",
  "cfg": 0,
  "steps": [
   1,
   4
  ],
  "seeds": [
   42,
   314,
   2026
  ],
  "tasks": [
   {
    "id": "product-photo",
    "use_case": "Generic product image for a small online shop",
    "prompt": "Studio product photograph of one plain white ceramic coffee mug on a pale beige tabletop, soft window light from the left, subtle shadow, uncluttered background, no letters, no logo."
   },
   {
    "id": "cafe-hero",
    "use_case": "Local cafe website hero artwork",
    "prompt": "Wide editorial photograph of a cozy neighborhood cafe interior with wooden tables, green plants and warm morning sunlight, the left side mostly an empty cream wall for a headline overlay, no people, no text."
   },
   {
    "id": "video-thumbnail",
    "use_case": "Original background for a technology review thumbnail",
    "prompt": "A compact silver desktop computer on a dark wooden desk, dramatic blue lighting on the right and orange lighting on the left, large empty dark space above the computer for title overlay, realistic photograph, no text, no logo."
   },
   {
    "id": "newsletter-illustration",
    "use_case": "Illustration for a practical home gardening newsletter",
    "prompt": "Clean watercolor illustration of three small herb plants in terracotta pots on a white kitchen windowsill, a small green watering can beside them, gentle natural colors, white background, no text."
   },
   {
    "id": "signage-text",
    "use_case": "Deliberately demanding exact-text sign for a shop",
    "prompt": "A simple front-facing rectangular cream shop sign with exactly the words OPEN TODAY in large dark green capital letters, centered, crisp readable typography, thin green border, no other words or decorations."
   }
  ],
  "limitations": "A compact established local diffusion model, not a claim about newest image-model quality. One model, five practical briefs, three fixed seeds. Text accuracy is a known difficult task and remains in the scored set. No cloud generation. No image editing or video generation measured by this phase.",
  "sources": [
   "https://github.com/ml-explore/mlx-examples/tree/042280ce50645b69d9c322ccf1cb8471384007f1/stable_diffusion",
   "https://huggingface.co/stabilityai/sdxl-turbo/tree/71153311d3dbb46851df1931d3ca6e939de83304"
  ]
 },
 "image_edit_settings": {
  "phase": "image-edit-v1",
  "model": "stabilityai/sdxl-turbo",
  "revision": "71153311d3dbb46851df1931d3ca6e939de83304",
  "runtime": "official ml-explore/mlx-examples stable_diffusion",
  "code_commit": "042280ce50645b69d9c322ccf1cb8471384007f1",
  "width": 512,
  "height": 512,
  "batch_size": 1,
  "unet_bits": 8,
  "unet_group_size": 32,
  "text_encoder_bits": 4,
  "text_encoder_group_size": 64,
  "vae_dtype": "float32",
  "cfg": 0,
  "steps": [
   4
  ],
  "seeds": [
   42,
   314,
   2026
  ],
  "tasks": [
   {
    "id": "product-photo",
    "use_case": "Image-to-image revision: Generic product image for a small online shop",
    "prompt": "Studio product photograph of one plain cobalt blue ceramic coffee mug on a pale beige tabletop, soft window light from the left, subtle shadow, uncluttered background, no letters, no logo.",
    "source_file": "product-photo.png",
    "source_sha256": "460d8c9c6c488aec55059e77cee3dc082c9e603247def04db5998582ba71d788",
    "original_prompt": "Studio product photograph of one plain white ceramic coffee mug on a pale beige tabletop, soft window light from the left, subtle shadow, uncluttered background, no letters, no logo."
   },
   {
    "id": "cafe-hero",
    "use_case": "Image-to-image revision: Local cafe website hero artwork",
    "prompt": "Photograph of the same cozy neighborhood cafe interior with wooden tables and green plants, now warmly lit in the evening by amber lamps, preserve the large blank cream wall on the left for a headline overlay, no people, no text.",
    "source_file": "cafe-hero.png",
    "source_sha256": "d95b9471650dc5fbc365f1b3c4ecdb98aa73495f5e351cb291e8af2c8743e65a",
    "original_prompt": "Wide editorial photograph of a cozy neighborhood cafe interior with wooden tables, green plants and warm morning sunlight, the left side mostly an empty cream wall for a headline overlay, no people, no text."
   },
   {
    "id": "video-thumbnail",
    "use_case": "Image-to-image revision: Original background for a technology review thumbnail",
    "prompt": "A compact silver desktop computer on the same dark wooden desk, dramatic purple lighting on the right and green lighting on the left, preserve large empty dark space above for a title overlay, realistic photograph, no text, no logo.",
    "source_file": "video-thumbnail.png",
    "source_sha256": "b376a69b0564fa98c0f298171b41abe29ba2a237e1625f3dea203a0f6c3d6bf9",
    "original_prompt": "A compact silver desktop computer on a dark wooden desk, dramatic blue lighting on the right and orange lighting on the left, large empty dark space above the computer for title overlay, realistic photograph, no text, no logo."
   },
   {
    "id": "newsletter-illustration",
    "use_case": "Image-to-image revision: Illustration for a practical home gardening newsletter",
    "prompt": "Clean black ink line drawing of three small herb plants in terracotta pots on a white kitchen windowsill, a small watering can beside them, monochrome white background, preserve the layout, no text.",
    "source_file": "newsletter-illustration.png",
    "source_sha256": "aadf96dddde8e28c52797526d976394a936336a8fd958e4882bbe68bd7cd5396",
    "original_prompt": "Clean watercolor illustration of three small herb plants in terracotta pots on a white kitchen windowsill, a small green watering can beside them, gentle natural colors, white background, no text."
   },
   {
    "id": "signage-text",
    "use_case": "Image-to-image revision: Deliberately demanding exact-text sign for a shop",
    "prompt": "The same front-facing rectangular cream shop sign, now with exactly the words CLOSED TODAY in large dark green capital letters, centered, crisp readable typography, preserve the thin green border, no other words or decorations.",
    "source_file": "signage-text.png",
    "source_sha256": "c5846fb8221904e0e7dbe6904b0af378eaeff20ccc5cef9c72ba1c2993df77c6",
    "original_prompt": "A simple front-facing rectangular cream shop sign with exactly the words OPEN TODAY in large dark green capital letters, centered, crisp readable typography, thin green border, no other words or decorations."
   }
  ],
  "limitations": "Plain SDXL Turbo image-to-image, not an instruction-tuned localized editor. Five fixed source images from the prior generation phase, same byte-identical sources on both hosts. Evaluate requested change AND source preservation separately. Nominal4steps with strength0.5 yields2actualdenoisingsteps. No video generation.",
  "sources": [
   "https://github.com/ml-explore/mlx-examples/tree/042280ce50645b69d9c322ccf1cb8471384007f1/stable_diffusion",
   "https://huggingface.co/stabilityai/sdxl-turbo/tree/71153311d3dbb46851df1931d3ca6e939de83304"
  ],
  "strength": 0.5,
  "actual_denoising_steps": 2
 },
 "judge": {
  "model": "gpt-6-astra",
  "reasoning_effort": "high"
 },
 "sources": {
  "mlx_lm": "https://github.com/ml-explore/mlx-lm",
  "image_runtime": "https://github.com/ml-explore/mlx-examples/tree/042280ce50645b69d9c322ccf1cb8471384007f1/stable_diffusion",
  "video_runtime": "https://github.com/imageio/imageio-ffmpeg",
  "speech_fixture": "https://huggingface.co/datasets/hf-internal-testing/librispeech_asr_dummy"
 },
 "limitations": [
  "No exhaustive maximum-model/context capacity sweep.",
  "No direct DRAM bandwidth, power or thermal benchmark.",
  "Generative video and real-world meeting transcription not measured.",
  "No throughput claim based on successful-work pairs where one host had no successful attempt."
 ]
}
