Adds support for Multimodal projector file (#104)

ilopezluna · web-flow · commit 9806a43e7961 · 2025-07-10T14:41:28.000+02:00
* Includes --proj if the model includes a Multimodal projector file

* Fix test
diff --git a/go.mod b/go.mod
@@ -5,7 +5,7 @@ go 1.23.7
 require (
 	github.com/containerd/containerd/v2 v2.0.4
 	github.com/containerd/platforms v1.0.0-rc.1
-	github.com/docker/model-distribution v0.0.0-20250627163720-aff34abcf3e0
+	github.com/docker/model-distribution v0.0.0-20250710064845-ad2903dc93f7
 	github.com/google/go-containerregistry v0.20.3
 	github.com/jaypipes/ghw v0.16.0
 	github.com/mattn/go-shellwords v1.0.12
diff --git a/go.sum b/go.sum
@@ -38,8 +38,8 @@ github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBi
 github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/docker-credential-helpers v0.8.2 h1:bX3YxiGzFP5sOXWc3bTPEXdEaZSeVMrFgOr3T+zrFAo=
 github.com/docker/docker-credential-helpers v0.8.2/go.mod h1:P3ci7E3lwkZg6XiHdRKft1KckHiO9a2rNtyFbZ/ry9M=
-github.com/docker/model-distribution v0.0.0-20250627163720-aff34abcf3e0 h1:bve4JZI06Admw+NewtPfrpJXsvRnGKTQvBOEICNC1C0=
-github.com/docker/model-distribution v0.0.0-20250627163720-aff34abcf3e0/go.mod h1:dThpO9JoG5Px3i+rTluAeZcqLGw8C0qepuEL4gL2o/c=
+github.com/docker/model-distribution v0.0.0-20250710064845-ad2903dc93f7 h1:Kkiv0E3LTbj2y3rPRIJtRf89Eahg372RxqvKV14zszo=
+github.com/docker/model-distribution v0.0.0-20250710064845-ad2903dc93f7/go.mod h1:dThpO9JoG5Px3i+rTluAeZcqLGw8C0qepuEL4gL2o/c=
 github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
diff --git a/pkg/inference/backends/llamacpp/llamacpp_config.go b/pkg/inference/backends/llamacpp/llamacpp_config.go
@@ -70,6 +70,12 @@ func (c *Config) GetArgs(model types.Model, socket string, mode inference.Backen
 		args = append(args, config.RuntimeFlags...)
 	}
 
+	// Add arguments for Multimodal projector
+	path, err := model.MMPROJPath()
+	if path != "" && err == nil {
+		args = append(args, "--mmproj", path)
+	}
+
 	return args, nil
 }
 
diff --git a/pkg/inference/backends/llamacpp/llamacpp_config_test.go b/pkg/inference/backends/llamacpp/llamacpp_config_test.go
@@ -1,6 +1,7 @@
 package llamacpp
 
 import (
+	"errors"
 	"runtime"
 	"strconv"
 	"testing"
@@ -251,6 +252,10 @@ type fakeModel struct {
 	config   types.Config
 }
 
+func (f *fakeModel) MMPROJPath() (string, error) {
+	return "", errors.New("not found")
+}
+
 func (f *fakeModel) ID() (string, error) {
 	panic("shouldn't be called")
 }
diff --git a/pkg/inference/models/manager.go b/pkg/inference/models/manager.go
@@ -333,7 +333,7 @@ func (m *Manager) handleDeleteModel(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
-	if err := m.distributionClient.DeleteModel(r.PathValue("name"), force); err != nil {
+	if _, err := m.distributionClient.DeleteModel(r.PathValue("name"), force); err != nil {
 		if errors.Is(err, distribution.ErrModelNotFound) {
 			http.Error(w, err.Error(), http.StatusNotFound)
 			return

Original file line number	Diff line number	Diff line change
`@@ -70,6 +70,12 @@ func (c *Config) GetArgs(model types.Model, socket string, mode inference.Backen`
`70`	`70`	`args = append(args, config.RuntimeFlags...)`
`71`	`71`	`}`
`72`	`72`
	`73`	`+ // Add arguments for Multimodal projector`
	`74`	`+ path, err := model.MMPROJPath()`
	`75`	`+ if path != "" && err == nil {`
	`76`	`+ args = append(args, "--mmproj", path)`
	`77`	`+ }`
	`78`	`+`
`73`	`79`	`return args, nil`
`74`	`80`	`}`
`75`	`81`
Original file line number	Diff line number	Diff line change
`@@ -333,7 +333,7 @@ func (m Manager) handleDeleteModel(w http.ResponseWriter, r http.Request) {`
`333`	`333`	`}`
`334`	`334`	`}`
`335`	`335`
`336`		`- if err := m.distributionClient.DeleteModel(r.PathValue("name"), force); err != nil {`
	`336`	`+ if _, err := m.distributionClient.DeleteModel(r.PathValue("name"), force); err != nil {`
`337`	`337`	`if errors.Is(err, distribution.ErrModelNotFound) {`
`338`	`338`	`http.Error(w, err.Error(), http.StatusNotFound)`
`339`	`339`	`return`