Skip to content

Commit 2c190ee

Browse files
mudlermudler
committed
feat(backends): add system backend, refactor
- Add a system backend path - Refactor and consolidate system information in system state - Use system state in all the components to figure out the system paths to used whenever needed - Refactor BackendConfig -> ModelConfig. This was otherway misleading as now we do have a backend configuration which is not the model config. Signed-off-by: Ettore Di Giacinto <[email protected]>
1 parent 19c92c7 commit 2c190ee

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

85 files changed

+999
-652
lines changed

‎core/application/application.go

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,22 +7,22 @@ import (
77
)
88

99
type Application struct {
10-
backendLoader *config.BackendConfigLoader
10+
backendLoader *config.ModelConfigLoader
1111
modelLoader *model.ModelLoader
1212
applicationConfig *config.ApplicationConfig
1313
templatesEvaluator *templates.Evaluator
1414
}
1515

1616
func newApplication(appConfig *config.ApplicationConfig) *Application {
1717
return &Application{
18-
backendLoader: config.NewBackendConfigLoader(appConfig.ModelPath),
19-
modelLoader: model.NewModelLoader(appConfig.ModelPath, appConfig.SingleBackend),
18+
backendLoader: config.NewModelConfigLoader(appConfig.SystemState.Model.ModelsPath),
19+
modelLoader: model.NewModelLoader(appConfig.SystemState, appConfig.SingleBackend),
2020
applicationConfig: appConfig,
21-
templatesEvaluator: templates.NewEvaluator(appConfig.ModelPath),
21+
templatesEvaluator: templates.NewEvaluator(appConfig.SystemState.Model.ModelsPath),
2222
}
2323
}
2424

25-
func (a *Application) BackendLoader() *config.BackendConfigLoader {
25+
func (a *Application) BackendLoader() *config.ModelConfigLoader {
2626
return a.backendLoader
2727
}
2828

‎core/application/startup.go

Lines changed: 15 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ func New(opts ...config.AppOption) (*Application, error) {
2020
options := config.NewApplicationConfig(opts...)
2121
application := newApplication(options)
2222

23-
log.Info().Msgf("Starting LocalAI using %d threads, with models path: %s", options.Threads, options.ModelPath)
23+
log.Info().Msgf("Starting LocalAI using %d threads, with models path: %s", options.Threads, options.SystemState.Model.ModelsPath)
2424
log.Info().Msgf("LocalAI version: %s", internal.PrintableVersion())
2525
caps, err := xsysinfo.CPUCapabilities()
2626
if err == nil {
@@ -35,10 +35,11 @@ func New(opts ...config.AppOption) (*Application, error) {
3535
}
3636

3737
// Make sure directories exists
38-
if options.ModelPath == "" {
39-
return nil, fmt.Errorf("options.ModelPath cannot be empty")
38+
if options.SystemState.Model.ModelsPath == "" {
39+
return nil, fmt.Errorf("models path cannot be empty")
4040
}
41-
err = os.MkdirAll(options.ModelPath, 0750)
41+
42+
err = os.MkdirAll(options.SystemState.Model.ModelsPath, 0750)
4243
if err != nil {
4344
return nil, fmt.Errorf("unable to create ModelPath: %q", err)
4445
}
@@ -55,50 +56,50 @@ func New(opts ...config.AppOption) (*Application, error) {
5556
}
5657
}
5758

58-
if err := coreStartup.InstallModels(options.Galleries, options.BackendGalleries, options.ModelPath, options.BackendsPath, options.EnforcePredownloadScans, options.AutoloadBackendGalleries, nil, options.ModelsURL...); err != nil {
59+
if err := coreStartup.InstallModels(options.Galleries, options.BackendGalleries, options.SystemState, options.EnforcePredownloadScans, options.AutoloadBackendGalleries, nil, options.ModelsURL...); err != nil {
5960
log.Error().Err(err).Msg("error installing models")
6061
}
6162

6263
for _, backend := range options.ExternalBackends {
63-
if err := coreStartup.InstallExternalBackends(options.BackendGalleries, options.BackendsPath, nil, backend, "", ""); err != nil {
64+
if err := coreStartup.InstallExternalBackends(options.BackendGalleries, options.SystemState, nil, backend, "", ""); err != nil {
6465
log.Error().Err(err).Msg("error installing external backend")
6566
}
6667
}
6768

6869
configLoaderOpts := options.ToConfigLoaderOptions()
6970

70-
if err := application.BackendLoader().LoadBackendConfigsFromPath(options.ModelPath, configLoaderOpts...); err != nil {
71+
if err := application.BackendLoader().LoadModelConfigsFromPath(options.SystemState.Model.ModelsPath, configLoaderOpts...); err != nil {
7172
log.Error().Err(err).Msg("error loading config files")
7273
}
7374

74-
if err := gallery.RegisterBackends(options.BackendsPath, application.ModelLoader()); err != nil {
75+
if err := gallery.RegisterBackends(options.SystemState, application.ModelLoader()); err != nil {
7576
log.Error().Err(err).Msg("error registering external backends")
7677
}
7778

7879
if options.ConfigFile != "" {
79-
if err := application.BackendLoader().LoadMultipleBackendConfigsSingleFile(options.ConfigFile, configLoaderOpts...); err != nil {
80+
if err := application.BackendLoader().LoadMultipleModelConfigsSingleFile(options.ConfigFile, configLoaderOpts...); err != nil {
8081
log.Error().Err(err).Msg("error loading config file")
8182
}
8283
}
8384

84-
if err := application.BackendLoader().Preload(options.ModelPath); err != nil {
85+
if err := application.BackendLoader().Preload(options.SystemState.Model.ModelsPath); err != nil {
8586
log.Error().Err(err).Msg("error downloading models")
8687
}
8788

8889
if options.PreloadJSONModels != "" {
89-
if err := services.ApplyGalleryFromString(options.ModelPath, options.BackendsPath, options.EnforcePredownloadScans, options.AutoloadBackendGalleries, options.Galleries, options.BackendGalleries, options.PreloadJSONModels); err != nil {
90+
if err := services.ApplyGalleryFromString(options.SystemState, options.EnforcePredownloadScans, options.AutoloadBackendGalleries, options.Galleries, options.BackendGalleries, options.PreloadJSONModels); err != nil {
9091
return nil, err
9192
}
9293
}
9394

9495
if options.PreloadModelsFromPath != "" {
95-
if err := services.ApplyGalleryFromFile(options.ModelPath, options.BackendsPath, options.EnforcePredownloadScans, options.AutoloadBackendGalleries, options.Galleries, options.BackendGalleries, options.PreloadModelsFromPath); err != nil {
96+
if err := services.ApplyGalleryFromFile(options.SystemState, options.EnforcePredownloadScans, options.AutoloadBackendGalleries, options.Galleries, options.BackendGalleries, options.PreloadModelsFromPath); err != nil {
9697
return nil, err
9798
}
9899
}
99100

100101
if options.Debug {
101-
for _, v := range application.BackendLoader().GetAllBackendConfigs() {
102+
for _, v := range application.BackendLoader().GetAllModelsConfigs() {
102103
log.Debug().Msgf("Model: %s (config: %+v)", v.Name, v)
103104
}
104105
}
@@ -131,7 +132,7 @@ func New(opts ...config.AppOption) (*Application, error) {
131132

132133
if options.LoadToMemory != nil && !options.SingleBackend {
133134
for _, m := range options.LoadToMemory {
134-
cfg, err := application.BackendLoader().LoadBackendConfigFileByNameDefaultOptions(m, options)
135+
cfg, err := application.BackendLoader().LoadModelConfigFileByNameDefaultOptions(m, options)
135136
if err != nil {
136137
return nil, err
137138
}

‎core/backend/detection.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,9 @@ func Detection(
1313
sourceFile string,
1414
loader *model.ModelLoader,
1515
appConfig *config.ApplicationConfig,
16-
backendConfig config.BackendConfig,
16+
modelConfig config.ModelConfig,
1717
) (*proto.DetectResponse, error) {
18-
opts := ModelOptions(backendConfig, appConfig)
18+
opts := ModelOptions(modelConfig, appConfig)
1919
detectionModel, err := loader.Load(opts...)
2020
if err != nil {
2121
return nil, err

‎core/backend/embeddings.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,9 @@ import (
99
model "github.com/mudler/LocalAI/pkg/model"
1010
)
1111

12-
func ModelEmbedding(s string, tokens []int, loader *model.ModelLoader, backendConfig config.BackendConfig, appConfig *config.ApplicationConfig) (func() ([]float32, error), error) {
12+
func ModelEmbedding(s string, tokens []int, loader *model.ModelLoader, modelConfig config.ModelConfig, appConfig *config.ApplicationConfig) (func() ([]float32, error), error) {
1313

14-
opts := ModelOptions(backendConfig, appConfig)
14+
opts := ModelOptions(modelConfig, appConfig)
1515

1616
inferenceModel, err := loader.Load(opts...)
1717
if err != nil {
@@ -23,7 +23,7 @@ func ModelEmbedding(s string, tokens []int, loader *model.ModelLoader, backendCo
2323
switch model := inferenceModel.(type) {
2424
case grpc.Backend:
2525
fn = func() ([]float32, error) {
26-
predictOptions := gRPCPredictOpts(backendConfig, loader.ModelPath)
26+
predictOptions := gRPCPredictOpts(modelConfig, loader.ModelPath)
2727
if len(tokens) > 0 {
2828
embeds := []int32{}
2929

‎core/backend/image.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,9 @@ import (
77
model "github.com/mudler/LocalAI/pkg/model"
88
)
99

10-
func ImageGeneration(height, width, mode, step, seed int, positive_prompt, negative_prompt, src, dst string, loader *model.ModelLoader, backendConfig config.BackendConfig, appConfig *config.ApplicationConfig, refImages []string) (func() error, error) {
10+
func ImageGeneration(height, width, mode, step, seed int, positive_prompt, negative_prompt, src, dst string, loader *model.ModelLoader, modelConfig config.ModelConfig, appConfig *config.ApplicationConfig, refImages []string) (func() error, error) {
1111

12-
opts := ModelOptions(backendConfig, appConfig)
12+
opts := ModelOptions(modelConfig, appConfig)
1313
inferenceModel, err := loader.Load(
1414
opts...,
1515
)
@@ -27,12 +27,12 @@ func ImageGeneration(height, width, mode, step, seed int, positive_prompt, negat
2727
Mode: int32(mode),
2828
Step: int32(step),
2929
Seed: int32(seed),
30-
CLIPSkip: int32(backendConfig.Diffusers.ClipSkip),
30+
CLIPSkip: int32(modelConfig.Diffusers.ClipSkip),
3131
PositivePrompt: positive_prompt,
3232
NegativePrompt: negative_prompt,
3333
Dst: dst,
3434
Src: src,
35-
EnableParameters: backendConfig.Diffusers.EnableParameters,
35+
EnableParameters: modelConfig.Diffusers.EnableParameters,
3636
RefImages: refImages,
3737
})
3838
return err

‎core/backend/llm.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ type TokenUsage struct {
3535
TimingTokenGeneration float64
3636
}
3737

38-
func ModelInference(ctx context.Context, s string, messages []schema.Message, images, videos, audios []string, loader *model.ModelLoader, c *config.BackendConfig, cl *config.BackendConfigLoader, o *config.ApplicationConfig, tokenCallback func(string, TokenUsage) bool) (func() (LLMResponse, error), error) {
38+
func ModelInference(ctx context.Context, s string, messages []schema.Message, images, videos, audios []string, loader *model.ModelLoader, c *config.ModelConfig, cl *config.ModelConfigLoader, o *config.ApplicationConfig, tokenCallback func(string, TokenUsage) bool) (func() (LLMResponse, error), error) {
3939
modelFile := c.Model
4040

4141
// Check if the modelFile exists, if it doesn't try to load it from the gallery
@@ -47,7 +47,7 @@ func ModelInference(ctx context.Context, s string, messages []schema.Message, im
4747
if !slices.Contains(modelNames, c.Name) {
4848
utils.ResetDownloadTimers()
4949
// if we failed to load the model, we try to download it
50-
err := gallery.InstallModelFromGallery(o.Galleries, o.BackendGalleries, c.Name, loader.ModelPath, o.BackendsPath, gallery.GalleryModel{}, utils.DisplayDownloadFunction, o.EnforcePredownloadScans, o.AutoloadBackendGalleries)
50+
err := gallery.InstallModelFromGallery(o.Galleries, o.BackendGalleries, o.SystemState, c.Name, gallery.GalleryModel{}, utils.DisplayDownloadFunction, o.EnforcePredownloadScans, o.AutoloadBackendGalleries)
5151
if err != nil {
5252
log.Error().Err(err).Msgf("failed to install model %q from gallery", modelFile)
5353
//return nil, err
@@ -201,7 +201,7 @@ func ModelInference(ctx context.Context, s string, messages []schema.Message, im
201201
var cutstrings map[string]*regexp.Regexp = make(map[string]*regexp.Regexp)
202202
var mu sync.Mutex = sync.Mutex{}
203203

204-
func Finetune(config config.BackendConfig, input, prediction string) string {
204+
func Finetune(config config.ModelConfig, input, prediction string) string {
205205
if config.Echo {
206206
prediction = input + prediction
207207
}

‎core/backend/llm_test.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,14 +12,14 @@ import (
1212
var _ = Describe("LLM tests", func() {
1313
Context("Finetune LLM output", func() {
1414
var (
15-
testConfig config.BackendConfig
15+
testConfig config.ModelConfig
1616
input string
1717
prediction string
1818
result string
1919
)
2020

2121
BeforeEach(func() {
22-
testConfig = config.BackendConfig{
22+
testConfig = config.ModelConfig{
2323
PredictionOptions: schema.PredictionOptions{
2424
Echo: false,
2525
},

‎core/backend/options.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ import (
1111
"github.com/rs/zerolog/log"
1212
)
1313

14-
func ModelOptions(c config.BackendConfig, so *config.ApplicationConfig, opts ...model.Option) []model.Option {
14+
func ModelOptions(c config.ModelConfig, so *config.ApplicationConfig, opts ...model.Option) []model.Option {
1515
name := c.Name
1616
if name == "" {
1717
name = c.Model
@@ -58,7 +58,7 @@ func ModelOptions(c config.BackendConfig, so *config.ApplicationConfig, opts ...
5858
return append(defOpts, opts...)
5959
}
6060

61-
func getSeed(c config.BackendConfig) int32 {
61+
func getSeed(c config.ModelConfig) int32 {
6262
var seed int32 = config.RAND_SEED
6363

6464
if c.Seed != nil {
@@ -72,7 +72,7 @@ func getSeed(c config.BackendConfig) int32 {
7272
return seed
7373
}
7474

75-
func grpcModelOpts(c config.BackendConfig) *pb.ModelOptions {
75+
func grpcModelOpts(c config.ModelConfig) *pb.ModelOptions {
7676
b := 512
7777
if c.Batch != 0 {
7878
b = c.Batch
@@ -195,7 +195,7 @@ func grpcModelOpts(c config.BackendConfig) *pb.ModelOptions {
195195
}
196196
}
197197

198-
func gRPCPredictOpts(c config.BackendConfig, modelPath string) *pb.PredictOptions {
198+
func gRPCPredictOpts(c config.ModelConfig, modelPath string) *pb.PredictOptions {
199199
promptCachePath := ""
200200
if c.PromptCachePath != "" {
201201
p := filepath.Join(modelPath, c.PromptCachePath)

‎core/backend/rerank.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,8 @@ import (
99
model "github.com/mudler/LocalAI/pkg/model"
1010
)
1111

12-
func Rerank(request *proto.RerankRequest, loader *model.ModelLoader, appConfig *config.ApplicationConfig, backendConfig config.BackendConfig) (*proto.RerankResult, error) {
13-
opts := ModelOptions(backendConfig, appConfig)
12+
func Rerank(request *proto.RerankRequest, loader *model.ModelLoader, appConfig *config.ApplicationConfig, modelConfig config.ModelConfig) (*proto.RerankResult, error) {
13+
opts := ModelOptions(modelConfig, appConfig)
1414
rerankModel, err := loader.Load(opts...)
1515
if err != nil {
1616
return nil, err

‎core/backend/soundgeneration.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,10 @@ func SoundGeneration(
2121
sourceDivisor *int32,
2222
loader *model.ModelLoader,
2323
appConfig *config.ApplicationConfig,
24-
backendConfig config.BackendConfig,
24+
modelConfig config.ModelConfig,
2525
) (string, *proto.Result, error) {
2626

27-
opts := ModelOptions(backendConfig, appConfig)
27+
opts := ModelOptions(modelConfig, appConfig)
2828
soundGenModel, err := loader.Load(opts...)
2929
if err != nil {
3030
return "", nil, err
@@ -49,7 +49,7 @@ func SoundGeneration(
4949

5050
res, err := soundGenModel.SoundGeneration(context.Background(), &proto.SoundGenerationRequest{
5151
Text: text,
52-
Model: backendConfig.Model,
52+
Model: modelConfig.Model,
5353
Dst: filePath,
5454
Sample: doSample,
5555
Duration: duration,

0 commit comments

Comments
 (0)