mudler--localai
wehub-resource-sync
55 行
1.7 KiB
Go
55 行
1.7 KiB
Go
package downloader
|
|
|
|
import (
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"strings"
|
|
|
|
"github.com/mudler/LocalAI/pkg/httpclient"
|
|
)
|
|
|
|
type HuggingFaceScanResult struct {
|
|
RepositoryId string `json:"repositoryId"`
|
|
Revision string `json:"revision"`
|
|
HasUnsafeFiles bool `json:"hasUnsafeFile"`
|
|
ClamAVInfectedFiles []string `json:"clamAVInfectedFiles"`
|
|
DangerousPickles []string `json:"dangerousPickles"`
|
|
ScansDone bool `json:"scansDone"`
|
|
}
|
|
|
|
var ErrNonHuggingFaceFile = errors.New("not a huggingface repo")
|
|
var ErrUnsafeFilesFound = errors.New("unsafe files found")
|
|
|
|
func HuggingFaceScan(uri URI) (*HuggingFaceScanResult, error) {
|
|
cleanParts := strings.Split(uri.ResolveURL(), "/")
|
|
// cleanParts[2] is the hostname from the URL (e.g. "huggingface.co" or "hf-mirror.com").
|
|
// Extract the hostname from HF_ENDPOINT for comparison, since HF_ENDPOINT includes the scheme.
|
|
hfHost := strings.TrimPrefix(strings.TrimPrefix(HF_ENDPOINT, "https://"), "http://")
|
|
if len(cleanParts) <= 4 || (cleanParts[2] != "huggingface.co" && cleanParts[2] != hfHost) {
|
|
return nil, ErrNonHuggingFaceFile
|
|
}
|
|
results, err := httpclient.New(httpclient.WithFollowRedirects()).Get(fmt.Sprintf("%s/api/models/%s/%s/scan", HF_ENDPOINT, cleanParts[3], cleanParts[4]))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer results.Body.Close()
|
|
if results.StatusCode != 200 {
|
|
return nil, fmt.Errorf("unexpected status code during HuggingFaceScan: %d", results.StatusCode)
|
|
}
|
|
scanResult := &HuggingFaceScanResult{}
|
|
bodyBytes, err := io.ReadAll(results.Body)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
err = json.Unmarshal(bodyBytes, scanResult)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if scanResult.HasUnsafeFiles {
|
|
return scanResult, ErrUnsafeFilesFound
|
|
}
|
|
return scanResult, nil
|
|
}
|