HEX
Server: Apache/2.4.54 (Win64) OpenSSL/1.1.1p PHP/7.4.30
System: Windows NT website-api 10.0 build 20348 (Windows Server 2016) AMD64
User: SYSTEM (0)
PHP: 7.4.30
Disabled: NONE
Upload Files
File: C:/github_repos/casibase_customer_0022/object/store.go
// Copyright 2023 The Casibase Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package object

import (
	"fmt"
	"strings"
	"time"

	"github.com/casibase/casibase/i18n"
	"github.com/casibase/casibase/storage"
	"github.com/casibase/casibase/util"
	"xorm.io/core"
)

type TreeFile struct {
	Key         string      `xorm:"varchar(100)" json:"key"`
	Title       string      `xorm:"varchar(100)" json:"title"`
	Size        int64       `json:"size"`
	CreatedTime string      `xorm:"varchar(100)" json:"createdTime"`
	IsLeaf      bool        `json:"isLeaf"`
	Url         string      `xorm:"varchar(255)" json:"url"`
	Children    []*TreeFile `xorm:"varchar(1000)" json:"children"`

	ChildrenMap map[string]*TreeFile `xorm:"-" json:"-"`
}

type Properties struct {
	CollectedTime string `xorm:"varchar(100)" json:"collectedTime"`
	Subject       string `xorm:"varchar(100)" json:"subject"`
}

type UsageInfo struct {
	Provider   string    `xorm:"varchar(100)" json:"provider"`
	TokenCount int       `json:"tokenCount"`
	StartTime  time.Time `xorm:"created" json:"startTime"`
}

type Prompt struct {
	Title string `json:"title"`
	Text  string `json:"text"`
	Image string `json:"image"`
}

type Store struct {
	Owner       string `xorm:"varchar(100) notnull pk" json:"owner"`
	Name        string `xorm:"varchar(100) notnull pk" json:"name"`
	CreatedTime string `xorm:"varchar(100)" json:"createdTime"`
	DisplayName string `xorm:"varchar(100)" json:"displayName"`

	StorageProvider      string   `xorm:"varchar(100)" json:"storageProvider"`
	StorageSubpath       string   `xorm:"varchar(100)" json:"storageSubpath"`
	ImageProvider        string   `xorm:"varchar(100)" json:"imageProvider"`
	SplitProvider        string   `xorm:"varchar(100)" json:"splitProvider"`
	SearchProvider       string   `xorm:"varchar(100)" json:"searchProvider"`
	ModelProvider        string   `xorm:"varchar(100)" json:"modelProvider"`
	EmbeddingProvider    string   `xorm:"varchar(100)" json:"embeddingProvider"`
	TextToSpeechProvider string   `xorm:"varchar(100)" json:"textToSpeechProvider"`
	EnableTtsStreaming   bool     `xorm:"bool" json:"enableTtsStreaming"`
	SpeechToTextProvider string   `xorm:"varchar(100)" json:"speechToTextProvider"`
	AgentProvider        string   `xorm:"varchar(100)" json:"agentProvider"`
	VectorStoreId        string   `xorm:"varchar(100)" json:"vectorStoreId"`
	BuiltinTools         []string `xorm:"varchar(500)" json:"builtinTools"`

	MemoryLimit         int      `json:"memoryLimit"`
	Frequency           int      `json:"frequency"`
	LimitMinutes        int      `json:"limitMinutes"`
	KnowledgeCount      int      `json:"knowledgeCount"`
	SuggestionCount     int      `json:"suggestionCount"`
	Welcome             string   `xorm:"varchar(100)" json:"welcome"`
	WelcomeTitle        string   `xorm:"varchar(100)" json:"welcomeTitle"`
	WelcomeText         string   `xorm:"varchar(100)" json:"welcomeText"`
	Prompt              string   `xorm:"mediumtext" json:"prompt"`
	Prompts             []Prompt `xorm:"mediumtext" json:"prompts"`
	ThemeColor          string   `xorm:"varchar(100)" json:"themeColor"`
	Avatar              string   `xorm:"varchar(200)" json:"avatar"`
	Title               string   `xorm:"varchar(100)" json:"title"`
	HtmlTitle           string   `xorm:"varchar(100)" json:"htmlTitle"`
	FaviconUrl          string   `xorm:"varchar(200)" json:"faviconUrl"`
	LogoUrl             string   `xorm:"varchar(200)" json:"logoUrl"`
	FooterHtml          string   `xorm:"mediumtext" json:"footerHtml"`
	NavItems            []string `xorm:"text" json:"navItems"`
	VectorStores        []string `xorm:"varchar(500)" json:"vectorStores"`
	ChildStores         []string `xorm:"varchar(500)" json:"childStores"`
	ChildModelProviders []string `xorm:"varchar(500)" json:"childModelProviders"`
	ForbiddenWords      []string `xorm:"text" json:"forbiddenWords"`
	ShowAutoRead        bool     `json:"showAutoRead"`
	DisableFileUpload   bool     `json:"disableFileUpload"`
	HideThinking        bool     `json:"hideThinking"`
	IsDefault           bool     `json:"isDefault"`
	State               string   `xorm:"varchar(100)" json:"state"`

	ChatCount    int `xorm:"-" json:"chatCount"`
	MessageCount int `xorm:"-" json:"messageCount"`

	FileTree      *TreeFile              `xorm:"mediumtext" json:"fileTree"`
	PropertiesMap map[string]*Properties `xorm:"mediumtext" json:"propertiesMap"`
}

func GetGlobalStores() ([]*Store, error) {
	stores := []*Store{}
	err := adapter.engine.Asc("owner").Desc("created_time").Find(&stores)
	if err != nil {
		return stores, err
	}

	return stores, nil
}

func GetStores(owner string) ([]*Store, error) {
	stores := []*Store{}
	err := adapter.engine.Desc("created_time").Find(&stores, &Store{Owner: owner})
	if err != nil {
		return stores, err
	}

	return stores, nil
}

func GetDefaultStore(owner string) (*Store, error) {
	stores, err := GetStores(owner)
	if err != nil {
		return nil, err
	}

	for _, store := range stores {
		if store.IsDefault {
			return store, nil
		}
	}

	for _, store := range stores {
		if store.State != "Inactive" && store.StorageProvider != "" && store.ModelProvider != "" && store.EmbeddingProvider != "" {
			return store, nil
		}
	}

	if len(stores) > 0 {
		return stores[0], nil
	}

	return nil, nil
}

func getStore(owner string, name string) (*Store, error) {
	store := Store{Owner: owner, Name: name}
	existed, err := adapter.engine.Get(&store)
	if err != nil {
		return &store, err
	}

	if existed {
		return &store, nil
	} else {
		return nil, nil
	}
}

func GetStore(id string) (*Store, error) {
	owner, name := util.GetOwnerAndNameFromId(id)
	return getStore(owner, name)
}

func UpdateStore(id string, store *Store) (bool, error) {
	owner, name := util.GetOwnerAndNameFromId(id)
	_, err := getStore(owner, name)
	if err != nil {
		return false, err
	}
	if store == nil {
		return false, nil
	}

	_, err = adapter.engine.ID(core.PK{owner, name}).AllCols().Update(store)
	if err != nil {
		return false, err
	}

	// return affected != 0
	return true, nil
}

func AddStore(store *Store) (bool, error) {
	affected, err := adapter.engine.Insert(store)
	if err != nil {
		return false, err
	}

	return affected != 0, nil
}

func DeleteStore(store *Store) (bool, error) {
	affected, err := adapter.engine.ID(core.PK{store.Owner, store.Name}).Delete(&Store{})
	if err != nil {
		return false, err
	}

	return affected != 0, nil
}

func (store *Store) GetId() string {
	return fmt.Sprintf("%s/%s", store.Owner, store.Name)
}

func (store *Store) GetStorageProviderObj(lang string) (storage.StorageProvider, error) {
	var provider *Provider
	var err error
	if store.StorageProvider == "" {
		provider, err = GetDefaultStorageProvider()
	} else {
		providerId := util.GetIdFromOwnerAndName(store.Owner, store.StorageProvider)
		provider, err = GetProvider(providerId)
	}
	if err != nil {
		return nil, err
	}

	var storageProvider storage.StorageProvider
	if provider != nil {
		storageProvider, err = provider.GetStorageProviderObj(store.VectorStoreId, lang)
		if err != nil {
			return nil, err
		}
	} else {
		storageProvider, err = storage.NewCasdoorProvider(store.StorageProvider, lang)
		if err != nil {
			return nil, err
		}
	}

	return NewSubpathStorageProvider(storageProvider, store.StorageSubpath), nil
}

func (store *Store) GetImageProviderObj(lang string) (storage.StorageProvider, error) {
	if store.ImageProvider == "" {
		return nil, fmt.Errorf(i18n.Translate(lang, "object:The image provider for store: %s should not be empty"), store.GetId())
	}

	return storage.NewCasdoorProvider(store.ImageProvider, lang)
}

func (store *Store) GetModelProvider() (*Provider, error) {
	if store.ModelProvider == "" {
		return GetDefaultModelProvider()
	}

	providerId := util.GetIdFromOwnerAndName(store.Owner, store.ModelProvider)
	return GetProvider(providerId)
}

func (store *Store) GetTextToSpeechProvider() (*Provider, error) {
	if store.TextToSpeechProvider == "" {
		return GetDefaultTextToSpeechProvider()
	}

	providerId := util.GetIdFromOwnerAndName(store.Owner, store.TextToSpeechProvider)
	return GetProvider(providerId)
}

func (store *Store) GetSpeechToTextProvider() (*Provider, error) {
	if store.SpeechToTextProvider == "" {
		return GetDefaultSpeechToTextProvider()
	}

	providerId := util.GetIdFromOwnerAndName(store.Owner, store.SpeechToTextProvider)
	return GetProvider(providerId)
}

func (store *Store) GetEmbeddingProvider() (*Provider, error) {
	if store.EmbeddingProvider == "" {
		return GetDefaultEmbeddingProvider()
	}

	providerId := util.GetIdFromOwnerAndName(store.Owner, store.EmbeddingProvider)
	return GetProvider(providerId)
}

func RefreshStoreVectors(store *Store, lang string) (bool, error) {
	storageProviderObj, err := store.GetStorageProviderObj(lang)
	if err != nil {
		return false, err
	}

	modelProvider, err := store.GetModelProvider()
	if err != nil {
		return false, err
	}
	if modelProvider == nil {
		return false, fmt.Errorf(i18n.Translate(lang, "object:The model provider for store: %s is not found"), store.GetId())
	}

	embeddingProvider, err := store.GetEmbeddingProvider()
	if err != nil {
		return false, err
	}
	if embeddingProvider == nil {
		return false, fmt.Errorf(i18n.Translate(lang, "object:The embedding provider for store: %s is not found"), store.GetId())
	}

	embeddingProviderObj, err := embeddingProvider.GetEmbeddingProvider(lang)
	if err != nil {
		return false, err
	}

	ok, err := addVectorsForStore(storageProviderObj, embeddingProviderObj, "", store.Name, store.SplitProvider, embeddingProvider.Name, modelProvider.SubType, lang)
	return ok, err
}

func refreshVector(vector *Vector, lang string) (bool, error) {
	_, embeddingProviderObj, err := getEmbeddingProviderFromName("admin", vector.Provider, lang)
	if err != nil {
		return false, err
	}

	data, _, err := queryVectorSafe(embeddingProviderObj, vector.Text, lang)
	if err != nil {
		return false, err
	}

	vector.Data = data

	return true, nil
}

func GetStoresByFields(owner string, fields ...string) ([]*Store, error) {
	var stores []*Store
	err := adapter.engine.Desc("created_time").Cols(fields...).Find(&stores, &Store{Owner: owner})
	if err != nil {
		return nil, err
	}

	return stores, nil
}

func GetStoreCount(name, field, value string) (int64, error) {
	session := GetDbSession("", -1, -1, field, value, "", "")
	return session.Count(&Store{Name: name})
}

func GetPaginationStores(offset, limit int, name, field, value, sortField, sortOrder string) ([]*Store, error) {
	stores := []*Store{}
	session := GetDbSession("", offset, limit, field, value, sortField, sortOrder)
	var err error
	if name != "" {
		err = session.Find(&stores, &Store{Name: name})
	} else {
		err = session.Find(&stores)
	}
	if err != nil {
		return stores, err
	}

	return stores, nil
}

func (store *Store) ContainsForbiddenWords(text string) (bool, string) {
	if store.ForbiddenWords == nil || len(store.ForbiddenWords) == 0 {
		return false, ""
	}

	lowerText := strings.ToLower(text)
	for _, forbiddenWord := range store.ForbiddenWords {
		if forbiddenWord == "" {
			continue
		}
		lowerForbiddenWord := strings.ToLower(forbiddenWord)
		if strings.Contains(lowerText, lowerForbiddenWord) {
			return true, forbiddenWord
		}
	}
	return false, ""
}