@@ -296,7 +296,7 @@ var _ = Describe("API test", func() {
296296 response := postModelApplyRequest ("http://127.0.0.1:9090/models/apply" , modelApplyRequest {
297297 URL : "github:go-skynet/model-gallery/openllama_3b.yaml" ,
298298 Name : "openllama_3b" ,
299- Overrides : map [string ]interface {}{"backend" : "llama" , "mmap" : true , "f16" : true , "context_size" : 128 },
299+ Overrides : map [string ]interface {}{"backend" : "llama-stable " , "mmap" : true , "f16" : true , "context_size" : 128 },
300300 })
301301
302302 Expect (response ["uuid" ]).ToNot (BeEmpty (), fmt .Sprint (response ))
@@ -359,6 +359,76 @@ var _ = Describe("API test", func() {
359359 Expect (string (resp2 .Choices [0 ].FinishReason )).To (Equal ("function_call" ), fmt .Sprint (resp2 .Choices [0 ].FinishReason ))
360360 })
361361
362+ It ("runs openllama gguf" , Label ("llama-gguf" ), func () {
363+ if runtime .GOOS != "linux" {
364+ Skip ("test supported only on linux" )
365+ }
366+ response := postModelApplyRequest ("http://127.0.0.1:9090/models/apply" , modelApplyRequest {
367+ URL : "github:go-skynet/model-gallery/openllama-3b-gguf.yaml" ,
368+ Name : "openllama_3b_gguf" ,
369+ Overrides : map [string ]interface {}{"backend" : "llama" , "mmap" : true , "f16" : true , "context_size" : 128 },
370+ })
371+
372+ Expect (response ["uuid" ]).ToNot (BeEmpty (), fmt .Sprint (response ))
373+
374+ uuid := response ["uuid" ].(string )
375+
376+ Eventually (func () bool {
377+ response := getModelStatus ("http://127.0.0.1:9090/models/jobs/" + uuid )
378+ return response ["processed" ].(bool )
379+ }, "360s" , "10s" ).Should (Equal (true ))
380+
381+ By ("testing completion" )
382+ resp , err := client .CreateCompletion (context .TODO (), openai.CompletionRequest {Model : "openllama_3b_gguf" , Prompt : "Count up to five: one, two, three, four, " })
383+ Expect (err ).ToNot (HaveOccurred ())
384+ Expect (len (resp .Choices )).To (Equal (1 ))
385+ Expect (resp .Choices [0 ].Text ).To (ContainSubstring ("five" ))
386+
387+ By ("testing functions" )
388+ resp2 , err := client .CreateChatCompletion (
389+ context .TODO (),
390+ openai.ChatCompletionRequest {
391+ Model : "openllama_3b_gguf" ,
392+ Messages : []openai.ChatCompletionMessage {
393+ {
394+ Role : "user" ,
395+ Content : "What is the weather like in San Francisco (celsius)?" ,
396+ },
397+ },
398+ Functions : []openai.FunctionDefinition {
399+ openai.FunctionDefinition {
400+ Name : "get_current_weather" ,
401+ Description : "Get the current weather" ,
402+ Parameters : jsonschema.Definition {
403+ Type : jsonschema .Object ,
404+ Properties : map [string ]jsonschema.Definition {
405+ "location" : {
406+ Type : jsonschema .String ,
407+ Description : "The city and state, e.g. San Francisco, CA" ,
408+ },
409+ "unit" : {
410+ Type : jsonschema .String ,
411+ Enum : []string {"celcius" , "fahrenheit" },
412+ },
413+ },
414+ Required : []string {"location" },
415+ },
416+ },
417+ },
418+ })
419+ Expect (err ).ToNot (HaveOccurred ())
420+ Expect (len (resp2 .Choices )).To (Equal (1 ))
421+ Expect (resp2 .Choices [0 ].Message .FunctionCall ).ToNot (BeNil ())
422+ Expect (resp2 .Choices [0 ].Message .FunctionCall .Name ).To (Equal ("get_current_weather" ), resp2 .Choices [0 ].Message .FunctionCall .Name )
423+
424+ var res map [string ]string
425+ err = json .Unmarshal ([]byte (resp2 .Choices [0 ].Message .FunctionCall .Arguments ), & res )
426+ Expect (err ).ToNot (HaveOccurred ())
427+ Expect (res ["location" ]).To (Equal ("San Francisco, California" ), fmt .Sprint (res ))
428+ Expect (res ["unit" ]).To (Equal ("celcius" ), fmt .Sprint (res ))
429+ Expect (string (resp2 .Choices [0 ].FinishReason )).To (Equal ("function_call" ), fmt .Sprint (resp2 .Choices [0 ].FinishReason ))
430+ })
431+
362432 It ("runs gpt4all" , Label ("gpt4all" ), func () {
363433 if runtime .GOOS != "linux" {
364434 Skip ("test supported only on linux" )
0 commit comments