From 90c46fb323b96265e23a00c7ddaa464e8130394b Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Thu, 12 Sep 2024 18:10:18 +0200 Subject: [PATCH 01/12] Cache library scan results --- internal/arduino/libraries/libraries.go | 231 ++++++++++++++++++ .../librariesmanager/librariesmanager.go | 86 +++++-- 2 files changed, 300 insertions(+), 17 deletions(-) diff --git a/internal/arduino/libraries/libraries.go b/internal/arduino/libraries/libraries.go index 9ffa1e27eea..a4ee7717d9f 100644 --- a/internal/arduino/libraries/libraries.go +++ b/internal/arduino/libraries/libraries.go @@ -16,8 +16,11 @@ package libraries import ( + "bytes" + "encoding/binary" "errors" "fmt" + "io" "github.com/arduino/arduino-cli/internal/arduino/cores" "github.com/arduino/arduino-cli/internal/arduino/globals" @@ -90,6 +93,234 @@ func (library *Library) String() string { return library.Name + "@" + library.Version.String() } +func (library *Library) MarshalBinary() []byte { + buffer := bytes.NewBuffer(make([]byte, 0, 4096)) + writeString := func(in string) { + inBytes := []byte(in) + binary.Write(buffer, binary.NativeEndian, uint16(len(inBytes))) + buffer.Write(inBytes) + } + writeStringArray := func(in []string) { + binary.Write(buffer, binary.NativeEndian, uint16(len(in))) + for _, i := range in { + writeString(i) + } + } + writeMap := func(in map[string]bool) { + binary.Write(buffer, binary.NativeEndian, uint16(len(in))) + for k, v := range in { + writeString(k) + binary.Write(buffer, binary.NativeEndian, v) + } + } + writePath := func(in *paths.Path) { + if in == nil { + writeString("") + } else { + writeString(in.String()) + } + } + writeString(library.Name) + writeString(library.Author) + writeString(library.Maintainer) + writeString(library.Sentence) + writeString(library.Paragraph) + writeString(library.Website) + writeString(library.Category) + writeStringArray(library.Architectures) + writeStringArray(library.Types) + writePath(library.InstallDir) + writeString(library.DirName) + writePath(library.SourceDir) + writePath(library.UtilityDir) + binary.Write(buffer, binary.NativeEndian, int32(library.Location)) + // library.ContainerPlatform *cores.PlatformRelease `json:""` + binary.Write(buffer, binary.NativeEndian, int32(library.Layout)) + binary.Write(buffer, binary.NativeEndian, library.DotALinkage) + binary.Write(buffer, binary.NativeEndian, library.Precompiled) + binary.Write(buffer, binary.NativeEndian, library.PrecompiledWithSources) + writeString(library.LDflags) + binary.Write(buffer, binary.NativeEndian, library.IsLegacy) + binary.Write(buffer, binary.NativeEndian, library.InDevelopment) + writeString(library.Version.String()) + writeString(library.License) + //writeStringArray(library.Properties.AsSlice()) + writeStringArray(library.Examples.AsStrings()) + writeStringArray(library.declaredHeaders) + writeStringArray(library.sourceHeaders) + writeMap(library.CompatibleWith) + return buffer.Bytes() +} + +func (library *Library) UnmarshalBinary(in io.Reader) error { + readString := func() (string, error) { + var len uint16 + if err := binary.Read(in, binary.NativeEndian, &len); err != nil { + return "", err + } + res := make([]byte, len) + if _, err := in.Read(res); err != nil { + return "", err + } + return string(res), nil + } + readStringArray := func() ([]string, error) { + var len uint16 + if err := binary.Read(in, binary.NativeEndian, &len); err != nil { + return nil, err + } + res := make([]string, len) + for i := range res { + var err error + res[i], err = readString() + if err != nil { + return nil, err + } + } + return res, nil + } + readMap := func() (map[string]bool, error) { + var len uint16 + if err := binary.Read(in, binary.NativeEndian, &len); err != nil { + return nil, err + } + res := map[string]bool{} + for i := uint16(0); i < len; i++ { + k, err := readString() + if err != nil { + return nil, err + } + var v bool + if err := binary.Read(in, binary.NativeEndian, &v); err != nil { + return nil, err + } + res[k] = v + } + return res, nil + } + var err error + library.Name, err = readString() + if err != nil { + return err + } + library.Author, err = readString() + if err != nil { + return err + } + library.Maintainer, err = readString() + if err != nil { + return err + } + library.Sentence, err = readString() + if err != nil { + return err + } + library.Paragraph, err = readString() + if err != nil { + return err + } + library.Website, err = readString() + if err != nil { + return err + } + library.Category, err = readString() + if err != nil { + return err + } + library.Architectures, err = readStringArray() + if err != nil { + return err + } + library.Types, err = readStringArray() + if err != nil { + return err + } + installDir, err := readString() + if err != nil { + return err + } + library.InstallDir = paths.New(installDir) + library.DirName, err = readString() + if err != nil { + return err + } + sourceDir, err := readString() + library.SourceDir = paths.New(sourceDir) + if err != nil { + return err + } + utilityDir, err := readString() + if err != nil { + return err + } + library.UtilityDir = paths.New(utilityDir) + var location int32 + if err := binary.Read(in, binary.NativeEndian, &location); err != nil { + return err + } + library.Location = LibraryLocation(location) + // library.ContainerPlatform *cores.PlatformRelease `json:""` + var layout int32 + if err := binary.Read(in, binary.NativeEndian, &layout); err != nil { + return err + } + library.Layout = LibraryLayout(layout) + if err := binary.Read(in, binary.NativeEndian, &library.DotALinkage); err != nil { + return err + } + if err := binary.Read(in, binary.NativeEndian, &library.Precompiled); err != nil { + return err + } + if err := binary.Read(in, binary.NativeEndian, &library.PrecompiledWithSources); err != nil { + return err + } + library.LDflags, err = readString() + if err != nil { + return err + } + if err := binary.Read(in, binary.NativeEndian, &library.IsLegacy); err != nil { + return err + } + if err := binary.Read(in, binary.NativeEndian, &library.InDevelopment); err != nil { + return err + } + version, err := readString() + if err != nil { + return err + } + library.Version = semver.MustParse(version) + library.License, err = readString() + if err != nil { + return err + } + // props, err := readStringArray() + // if err != nil { + // return err + // } + // library.Properties, err = properties.LoadFromSlice(props) + // if err != nil { + // return err + // } + examples, err := readStringArray() + if err != nil { + return err + } + library.Examples = paths.NewPathList(examples...) + library.declaredHeaders, err = readStringArray() + if err != nil { + return err + } + library.sourceHeaders, err = readStringArray() + if err != nil { + return err + } + library.CompatibleWith, err = readMap() + if err != nil { + return err + } + return nil +} + // ToRPCLibrary converts this library into an rpc.Library func (library *Library) ToRPCLibrary() (*rpc.Library, error) { pathOrEmpty := func(p *paths.Path) string { diff --git a/internal/arduino/libraries/librariesmanager/librariesmanager.go b/internal/arduino/libraries/librariesmanager/librariesmanager.go index 457d20b32ff..46cc68660eb 100644 --- a/internal/arduino/libraries/librariesmanager/librariesmanager.go +++ b/internal/arduino/libraries/librariesmanager/librariesmanager.go @@ -16,6 +16,7 @@ package librariesmanager import ( + "encoding/binary" "errors" "fmt" "os" @@ -201,30 +202,81 @@ func (lm *LibrariesManager) loadLibrariesFromDir(librariesDir *LibrariesDir) []* librariesDir.scanned = true - var libDirs paths.PathList - if librariesDir.IsSingleLibrary { - libDirs.Add(librariesDir.Path) + var loadedLibs []*libraries.Library + if cacheFilePath := librariesDir.Path.Join("libraries-loader-cache"); cacheFilePath.Exist() { + logrus.WithField("file", cacheFilePath).Info("Using library cache") + + // Load lib cache + cache, err := cacheFilePath.Open() + if err != nil { + s := status.Newf(codes.FailedPrecondition, "reading lib cache %[1]s: %[2]s", cacheFilePath, err) + return append(statuses, s) + } + defer cache.Close() + var n int32 + if err := binary.Read(cache, binary.NativeEndian, &n); err != nil { + s := status.Newf(codes.FailedPrecondition, "reading lib cache %[1]s: %[2]s", cacheFilePath, err) + return append(statuses, s) + } + loadedLibs = make([]*libraries.Library, n) + for i := range loadedLibs { + var lib libraries.Library + if err := lib.UnmarshalBinary(cache); err != nil { + s := status.Newf(codes.FailedPrecondition, "reading lib cache %[1]s: %[2]s", cacheFilePath, err) + return append(statuses, s) + } + loadedLibs[i] = &lib + } } else { - d, err := librariesDir.Path.ReadDir() - if os.IsNotExist(err) { - return statuses + var libDirs paths.PathList + if librariesDir.IsSingleLibrary { + libDirs.Add(librariesDir.Path) + } else { + d, err := librariesDir.Path.ReadDir() + if os.IsNotExist(err) { + return statuses + } + if err != nil { + s := status.Newf(codes.FailedPrecondition, i18n.Tr("reading dir %[1]s: %[2]s", librariesDir.Path, err)) + return append(statuses, s) + } + d.FilterDirs() + d.FilterOutHiddenFiles() + libDirs = d } + + for _, libDir := range libDirs { + library, err := libraries.Load(libDir, librariesDir.Location) + if err != nil { + s := status.Newf(codes.Internal, i18n.Tr("loading library from %[1]s: %[2]s", libDir, err)) + statuses = append(statuses, s) + continue + } + loadedLibs = append(loadedLibs, library) + } + + // Write lib cache + cache, err := cacheFilePath.Create() if err != nil { - s := status.New(codes.FailedPrecondition, i18n.Tr("reading dir %[1]s: %[2]s", librariesDir.Path, err)) + s := status.Newf(codes.FailedPrecondition, "writing lib cache %[1]s: %[2]s", cacheFilePath, err) + return append(statuses, s) + } + defer cache.Close() + if err := binary.Write(cache, binary.NativeEndian, int32(len(loadedLibs))); err != nil { + cacheFilePath.Remove() + s := status.Newf(codes.FailedPrecondition, "writing lib cache %[1]s: %[2]s", cacheFilePath, err) return append(statuses, s) } - d.FilterDirs() - d.FilterOutHiddenFiles() - libDirs = d + for _, lib := range loadedLibs { + if _, err := cache.Write(lib.MarshalBinary()); err != nil { + cacheFilePath.Remove() + s := status.Newf(codes.FailedPrecondition, "writing lib cache %[1]s: %[2]s", cacheFilePath, err) + return append(statuses, s) + } + } } - for _, libDir := range libDirs { - library, err := libraries.Load(libDir, librariesDir.Location) - if err != nil { - s := status.New(codes.Internal, i18n.Tr("loading library from %[1]s: %[2]s", libDir, err)) - statuses = append(statuses, s) - continue - } + for _, library := range loadedLibs { library.ContainerPlatform = librariesDir.PlatformRelease alternatives := lm.libraries[library.Name] alternatives.Add(library) From e973cc93b102861c30581b81cf0b676e86292401 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Fri, 13 Sep 2024 17:52:00 +0200 Subject: [PATCH 02/12] Added error check on cache write --- internal/arduino/libraries/libraries.go | 153 +++++++++++++----- .../librariesmanager/librariesmanager.go | 6 +- 2 files changed, 116 insertions(+), 43 deletions(-) diff --git a/internal/arduino/libraries/libraries.go b/internal/arduino/libraries/libraries.go index a4ee7717d9f..e9886abf091 100644 --- a/internal/arduino/libraries/libraries.go +++ b/internal/arduino/libraries/libraries.go @@ -93,63 +93,132 @@ func (library *Library) String() string { return library.Name + "@" + library.Version.String() } -func (library *Library) MarshalBinary() []byte { +func (library *Library) MarshalBinary() ([]byte, error) { buffer := bytes.NewBuffer(make([]byte, 0, 4096)) - writeString := func(in string) { + writeString := func(in string) error { inBytes := []byte(in) - binary.Write(buffer, binary.NativeEndian, uint16(len(inBytes))) - buffer.Write(inBytes) + if err := binary.Write(buffer, binary.NativeEndian, uint16(len(inBytes))); err != nil { + return err + } + _, err := buffer.Write(inBytes) + return err } - writeStringArray := func(in []string) { - binary.Write(buffer, binary.NativeEndian, uint16(len(in))) + writeStringArray := func(in []string) error { + if err := binary.Write(buffer, binary.NativeEndian, uint16(len(in))); err != nil { + return err + } for _, i := range in { - writeString(i) + if err := writeString(i); err != nil { + return err + } } + return nil } - writeMap := func(in map[string]bool) { - binary.Write(buffer, binary.NativeEndian, uint16(len(in))) + writeMap := func(in map[string]bool) error { + if err := binary.Write(buffer, binary.NativeEndian, uint16(len(in))); err != nil { + return err + } for k, v := range in { - writeString(k) - binary.Write(buffer, binary.NativeEndian, v) + if err := writeString(k); err != nil { + return err + } + if err := binary.Write(buffer, binary.NativeEndian, v); err != nil { + return err + } } + return nil } - writePath := func(in *paths.Path) { + writePath := func(in *paths.Path) error { if in == nil { - writeString("") + return writeString("") } else { - writeString(in.String()) + return writeString(in.String()) } } - writeString(library.Name) - writeString(library.Author) - writeString(library.Maintainer) - writeString(library.Sentence) - writeString(library.Paragraph) - writeString(library.Website) - writeString(library.Category) - writeStringArray(library.Architectures) - writeStringArray(library.Types) - writePath(library.InstallDir) - writeString(library.DirName) - writePath(library.SourceDir) - writePath(library.UtilityDir) - binary.Write(buffer, binary.NativeEndian, int32(library.Location)) + if err := writeString(library.Name); err != nil { + return nil, err + } + if err := writeString(library.Author); err != nil { + return nil, err + } + if err := writeString(library.Maintainer); err != nil { + return nil, err + } + if err := writeString(library.Sentence); err != nil { + return nil, err + } + if err := writeString(library.Paragraph); err != nil { + return nil, err + } + if err := writeString(library.Website); err != nil { + return nil, err + } + if err := writeString(library.Category); err != nil { + return nil, err + } + if err := writeStringArray(library.Architectures); err != nil { + return nil, err + } + if err := writeStringArray(library.Types); err != nil { + return nil, err + } + if err := writePath(library.InstallDir); err != nil { + return nil, err + } + if err := writeString(library.DirName); err != nil { + return nil, err + } + if err := writePath(library.SourceDir); err != nil { + return nil, err + } + if err := writePath(library.UtilityDir); err != nil { + return nil, err + } + if err := binary.Write(buffer, binary.NativeEndian, int32(library.Location)); err != nil { + return nil, err + } // library.ContainerPlatform *cores.PlatformRelease `json:""` - binary.Write(buffer, binary.NativeEndian, int32(library.Layout)) - binary.Write(buffer, binary.NativeEndian, library.DotALinkage) - binary.Write(buffer, binary.NativeEndian, library.Precompiled) - binary.Write(buffer, binary.NativeEndian, library.PrecompiledWithSources) - writeString(library.LDflags) - binary.Write(buffer, binary.NativeEndian, library.IsLegacy) - binary.Write(buffer, binary.NativeEndian, library.InDevelopment) - writeString(library.Version.String()) - writeString(library.License) + if err := binary.Write(buffer, binary.NativeEndian, int32(library.Layout)); err != nil { + return nil, err + } + if err := binary.Write(buffer, binary.NativeEndian, library.DotALinkage); err != nil { + return nil, err + } + if err := binary.Write(buffer, binary.NativeEndian, library.Precompiled); err != nil { + return nil, err + } + if err := binary.Write(buffer, binary.NativeEndian, library.PrecompiledWithSources); err != nil { + return nil, err + } + if err := writeString(library.LDflags); err != nil { + return nil, err + } + if err := binary.Write(buffer, binary.NativeEndian, library.IsLegacy); err != nil { + return nil, err + } + if err := binary.Write(buffer, binary.NativeEndian, library.InDevelopment); err != nil { + return nil, err + } + if err := writeString(library.Version.String()); err != nil { + return nil, err + } + if err := writeString(library.License); err != nil { + return nil, err + } //writeStringArray(library.Properties.AsSlice()) - writeStringArray(library.Examples.AsStrings()) - writeStringArray(library.declaredHeaders) - writeStringArray(library.sourceHeaders) - writeMap(library.CompatibleWith) - return buffer.Bytes() + if err := writeStringArray(library.Examples.AsStrings()); err != nil { + return nil, err + } + if err := writeStringArray(library.declaredHeaders); err != nil { + return nil, err + } + if err := writeStringArray(library.sourceHeaders); err != nil { + return nil, err + } + if err := writeMap(library.CompatibleWith); err != nil { + return nil, err + } + return buffer.Bytes(), nil } func (library *Library) UnmarshalBinary(in io.Reader) error { diff --git a/internal/arduino/libraries/librariesmanager/librariesmanager.go b/internal/arduino/libraries/librariesmanager/librariesmanager.go index 46cc68660eb..d7ca6567c17 100644 --- a/internal/arduino/libraries/librariesmanager/librariesmanager.go +++ b/internal/arduino/libraries/librariesmanager/librariesmanager.go @@ -268,7 +268,11 @@ func (lm *LibrariesManager) loadLibrariesFromDir(librariesDir *LibrariesDir) []* return append(statuses, s) } for _, lib := range loadedLibs { - if _, err := cache.Write(lib.MarshalBinary()); err != nil { + data, err := lib.MarshalBinary() + if err != nil { + panic("could not encode lib data for: " + lib.InstallDir.String()) + } + if _, err := cache.Write(data); err != nil { cacheFilePath.Remove() s := status.Newf(codes.FailedPrecondition, "writing lib cache %[1]s: %[2]s", cacheFilePath, err) return append(statuses, s) From f6d928676dc56a9f6a8d989293fc1eb1edf69fcc Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Thu, 19 Sep 2024 16:23:33 +0200 Subject: [PATCH 03/12] Using LibrariesList / refactored BinaryMarshal metods --- internal/arduino/libraries/libraries.go | 84 +++++++++---------- internal/arduino/libraries/librarieslist.go | 32 +++++++ .../librariesmanager/librariesmanager.go | 34 ++------ 3 files changed, 80 insertions(+), 70 deletions(-) diff --git a/internal/arduino/libraries/libraries.go b/internal/arduino/libraries/libraries.go index e9886abf091..d56e479a403 100644 --- a/internal/arduino/libraries/libraries.go +++ b/internal/arduino/libraries/libraries.go @@ -16,7 +16,6 @@ package libraries import ( - "bytes" "encoding/binary" "errors" "fmt" @@ -93,18 +92,17 @@ func (library *Library) String() string { return library.Name + "@" + library.Version.String() } -func (library *Library) MarshalBinary() ([]byte, error) { - buffer := bytes.NewBuffer(make([]byte, 0, 4096)) +func (library *Library) MarshalBinary(out io.Writer) error { writeString := func(in string) error { inBytes := []byte(in) - if err := binary.Write(buffer, binary.NativeEndian, uint16(len(inBytes))); err != nil { + if err := binary.Write(out, binary.NativeEndian, uint16(len(inBytes))); err != nil { return err } - _, err := buffer.Write(inBytes) + _, err := out.Write(inBytes) return err } writeStringArray := func(in []string) error { - if err := binary.Write(buffer, binary.NativeEndian, uint16(len(in))); err != nil { + if err := binary.Write(out, binary.NativeEndian, uint16(len(in))); err != nil { return err } for _, i := range in { @@ -115,14 +113,14 @@ func (library *Library) MarshalBinary() ([]byte, error) { return nil } writeMap := func(in map[string]bool) error { - if err := binary.Write(buffer, binary.NativeEndian, uint16(len(in))); err != nil { + if err := binary.Write(out, binary.NativeEndian, uint16(len(in))); err != nil { return err } for k, v := range in { if err := writeString(k); err != nil { return err } - if err := binary.Write(buffer, binary.NativeEndian, v); err != nil { + if err := binary.Write(out, binary.NativeEndian, v); err != nil { return err } } @@ -136,89 +134,89 @@ func (library *Library) MarshalBinary() ([]byte, error) { } } if err := writeString(library.Name); err != nil { - return nil, err + return err } if err := writeString(library.Author); err != nil { - return nil, err + return err } if err := writeString(library.Maintainer); err != nil { - return nil, err + return err } if err := writeString(library.Sentence); err != nil { - return nil, err + return err } if err := writeString(library.Paragraph); err != nil { - return nil, err + return err } if err := writeString(library.Website); err != nil { - return nil, err + return err } if err := writeString(library.Category); err != nil { - return nil, err + return err } if err := writeStringArray(library.Architectures); err != nil { - return nil, err + return err } if err := writeStringArray(library.Types); err != nil { - return nil, err + return err } if err := writePath(library.InstallDir); err != nil { - return nil, err + return err } if err := writeString(library.DirName); err != nil { - return nil, err + return err } if err := writePath(library.SourceDir); err != nil { - return nil, err + return err } if err := writePath(library.UtilityDir); err != nil { - return nil, err + return err } - if err := binary.Write(buffer, binary.NativeEndian, int32(library.Location)); err != nil { - return nil, err + if err := binary.Write(out, binary.NativeEndian, int32(library.Location)); err != nil { + return err } // library.ContainerPlatform *cores.PlatformRelease `json:""` - if err := binary.Write(buffer, binary.NativeEndian, int32(library.Layout)); err != nil { - return nil, err + if err := binary.Write(out, binary.NativeEndian, int32(library.Layout)); err != nil { + return err } - if err := binary.Write(buffer, binary.NativeEndian, library.DotALinkage); err != nil { - return nil, err + if err := binary.Write(out, binary.NativeEndian, library.DotALinkage); err != nil { + return err } - if err := binary.Write(buffer, binary.NativeEndian, library.Precompiled); err != nil { - return nil, err + if err := binary.Write(out, binary.NativeEndian, library.Precompiled); err != nil { + return err } - if err := binary.Write(buffer, binary.NativeEndian, library.PrecompiledWithSources); err != nil { - return nil, err + if err := binary.Write(out, binary.NativeEndian, library.PrecompiledWithSources); err != nil { + return err } if err := writeString(library.LDflags); err != nil { - return nil, err + return err } - if err := binary.Write(buffer, binary.NativeEndian, library.IsLegacy); err != nil { - return nil, err + if err := binary.Write(out, binary.NativeEndian, library.IsLegacy); err != nil { + return err } - if err := binary.Write(buffer, binary.NativeEndian, library.InDevelopment); err != nil { - return nil, err + if err := binary.Write(out, binary.NativeEndian, library.InDevelopment); err != nil { + return err } if err := writeString(library.Version.String()); err != nil { - return nil, err + return err } if err := writeString(library.License); err != nil { - return nil, err + return err } //writeStringArray(library.Properties.AsSlice()) if err := writeStringArray(library.Examples.AsStrings()); err != nil { - return nil, err + return err } if err := writeStringArray(library.declaredHeaders); err != nil { - return nil, err + return err } if err := writeStringArray(library.sourceHeaders); err != nil { - return nil, err + return err } if err := writeMap(library.CompatibleWith); err != nil { - return nil, err + return err } - return buffer.Bytes(), nil + return nil } func (library *Library) UnmarshalBinary(in io.Reader) error { diff --git a/internal/arduino/libraries/librarieslist.go b/internal/arduino/libraries/librarieslist.go index 84062203b76..032c110cd14 100644 --- a/internal/arduino/libraries/librarieslist.go +++ b/internal/arduino/libraries/librarieslist.go @@ -16,6 +16,9 @@ package libraries import ( + "encoding/binary" + "fmt" + "io" "sort" semver "go.bug.st/relaxed-semver" @@ -41,6 +44,35 @@ func (list *List) Add(libs ...*Library) { } } +func (list *List) UnmarshalBinary(in io.Reader) error { + var n int32 + if err := binary.Read(in, binary.NativeEndian, &n); err != nil { + return err + } + res := make([]*Library, n) + for i := range res { + var lib Library + if err := lib.UnmarshalBinary(in); err != nil { + return err + } + res[i] = &lib + } + *list = res + return nil +} + +func (list *List) MarshalBinary(out io.Writer) error { + if err := binary.Write(out, binary.NativeEndian, int32(len(*list))); err != nil { + return err + } + for _, lib := range *list { + if err := lib.MarshalBinary(out); err != nil { + return fmt.Errorf("could not encode lib data of %s: %w", lib.InstallDir.String(), err) + } + } + return nil +} + // Remove removes the given library from the list func (list *List) Remove(libraryToRemove *Library) { for i, lib := range *list { diff --git a/internal/arduino/libraries/librariesmanager/librariesmanager.go b/internal/arduino/libraries/librariesmanager/librariesmanager.go index d7ca6567c17..6eba79c4e8d 100644 --- a/internal/arduino/libraries/librariesmanager/librariesmanager.go +++ b/internal/arduino/libraries/librariesmanager/librariesmanager.go @@ -16,7 +16,6 @@ package librariesmanager import ( - "encoding/binary" "errors" "fmt" "os" @@ -202,7 +201,7 @@ func (lm *LibrariesManager) loadLibrariesFromDir(librariesDir *LibrariesDir) []* librariesDir.scanned = true - var loadedLibs []*libraries.Library + var loadedLibs libraries.List if cacheFilePath := librariesDir.Path.Join("libraries-loader-cache"); cacheFilePath.Exist() { logrus.WithField("file", cacheFilePath).Info("Using library cache") @@ -213,20 +212,11 @@ func (lm *LibrariesManager) loadLibrariesFromDir(librariesDir *LibrariesDir) []* return append(statuses, s) } defer cache.Close() - var n int32 - if err := binary.Read(cache, binary.NativeEndian, &n); err != nil { + + if err := loadedLibs.UnmarshalBinary(cache); err != nil { s := status.Newf(codes.FailedPrecondition, "reading lib cache %[1]s: %[2]s", cacheFilePath, err) return append(statuses, s) } - loadedLibs = make([]*libraries.Library, n) - for i := range loadedLibs { - var lib libraries.Library - if err := lib.UnmarshalBinary(cache); err != nil { - s := status.Newf(codes.FailedPrecondition, "reading lib cache %[1]s: %[2]s", cacheFilePath, err) - return append(statuses, s) - } - loadedLibs[i] = &lib - } } else { var libDirs paths.PathList if librariesDir.IsSingleLibrary { @@ -258,26 +248,16 @@ func (lm *LibrariesManager) loadLibrariesFromDir(librariesDir *LibrariesDir) []* // Write lib cache cache, err := cacheFilePath.Create() if err != nil { - s := status.Newf(codes.FailedPrecondition, "writing lib cache %[1]s: %[2]s", cacheFilePath, err) + s := status.Newf(codes.FailedPrecondition, "creating lib cache %[1]s: %[2]s", cacheFilePath, err) return append(statuses, s) } - defer cache.Close() - if err := binary.Write(cache, binary.NativeEndian, int32(len(loadedLibs))); err != nil { + err = loadedLibs.MarshalBinary(cache) + cache.Close() + if err != nil { cacheFilePath.Remove() s := status.Newf(codes.FailedPrecondition, "writing lib cache %[1]s: %[2]s", cacheFilePath, err) return append(statuses, s) } - for _, lib := range loadedLibs { - data, err := lib.MarshalBinary() - if err != nil { - panic("could not encode lib data for: " + lib.InstallDir.String()) - } - if _, err := cache.Write(data); err != nil { - cacheFilePath.Remove() - s := status.Newf(codes.FailedPrecondition, "writing lib cache %[1]s: %[2]s", cacheFilePath, err) - return append(statuses, s) - } - } } for _, library := range loadedLibs { From 6756e267134042a39797bddb434d74f5c6b9b741 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Fri, 20 Sep 2024 16:26:36 +0200 Subject: [PATCH 04/12] Made the lib cache files re-locatable --- internal/arduino/libraries/libraries.go | 57 +++++++++++++++---- internal/arduino/libraries/librarieslist.go | 9 +-- .../librariesmanager/librariesmanager.go | 4 +- 3 files changed, 52 insertions(+), 18 deletions(-) diff --git a/internal/arduino/libraries/libraries.go b/internal/arduino/libraries/libraries.go index d56e479a403..3cee3afd046 100644 --- a/internal/arduino/libraries/libraries.go +++ b/internal/arduino/libraries/libraries.go @@ -92,7 +92,7 @@ func (library *Library) String() string { return library.Name + "@" + library.Version.String() } -func (library *Library) MarshalBinary(out io.Writer) error { +func (library *Library) MarshalBinary(out io.Writer, prefix *paths.Path) error { writeString := func(in string) error { inBytes := []byte(in) if err := binary.Write(out, binary.NativeEndian, uint16(len(inBytes))); err != nil { @@ -129,10 +129,23 @@ func (library *Library) MarshalBinary(out io.Writer) error { writePath := func(in *paths.Path) error { if in == nil { return writeString("") + } else if p, err := in.RelFrom(prefix); err != nil { + return err } else { - return writeString(in.String()) + return writeString(p.String()) } } + writePathList := func(in []*paths.Path) error { + if err := binary.Write(out, binary.NativeEndian, uint16(len(in))); err != nil { + return err + } + for _, p := range in { + if err := writePath(p); err != nil { + return err + } + } + return nil + } if err := writeString(library.Name); err != nil { return err } @@ -204,7 +217,7 @@ func (library *Library) MarshalBinary(out io.Writer) error { return err } //writeStringArray(library.Properties.AsSlice()) - if err := writeStringArray(library.Examples.AsStrings()); err != nil { + if err := writePathList(library.Examples); err != nil { return err } if err := writeStringArray(library.declaredHeaders); err != nil { @@ -219,7 +232,7 @@ func (library *Library) MarshalBinary(out io.Writer) error { return nil } -func (library *Library) UnmarshalBinary(in io.Reader) error { +func (library *Library) UnmarshalBinary(in io.Reader, prefix *paths.Path) error { readString := func() (string, error) { var len uint16 if err := binary.Read(in, binary.NativeEndian, &len); err != nil { @@ -265,6 +278,30 @@ func (library *Library) UnmarshalBinary(in io.Reader) error { } return res, nil } + readPath := func() (*paths.Path, error) { + if p, err := readString(); err != nil { + return nil, err + } else if p == "" { + return nil, nil + } else { + return prefix.Join(p), nil + } + } + readPathList := func() (paths.PathList, error) { + var len uint16 + if err := binary.Read(in, binary.NativeEndian, &len); err != nil { + return nil, err + } + list := paths.NewPathList() + for range len { + if p, err := readPath(); err != nil { + return nil, err + } else { + list.Add(p) + } + } + return list, nil + } var err error library.Name, err = readString() if err != nil { @@ -302,25 +339,22 @@ func (library *Library) UnmarshalBinary(in io.Reader) error { if err != nil { return err } - installDir, err := readString() + library.InstallDir, err = readPath() if err != nil { return err } - library.InstallDir = paths.New(installDir) library.DirName, err = readString() if err != nil { return err } - sourceDir, err := readString() - library.SourceDir = paths.New(sourceDir) + library.SourceDir, err = readPath() if err != nil { return err } - utilityDir, err := readString() + library.UtilityDir, err = readPath() if err != nil { return err } - library.UtilityDir = paths.New(utilityDir) var location int32 if err := binary.Read(in, binary.NativeEndian, &location); err != nil { return err @@ -368,11 +402,10 @@ func (library *Library) UnmarshalBinary(in io.Reader) error { // if err != nil { // return err // } - examples, err := readStringArray() + library.Examples, err = readPathList() if err != nil { return err } - library.Examples = paths.NewPathList(examples...) library.declaredHeaders, err = readStringArray() if err != nil { return err diff --git a/internal/arduino/libraries/librarieslist.go b/internal/arduino/libraries/librarieslist.go index 032c110cd14..ff21a76bdb8 100644 --- a/internal/arduino/libraries/librarieslist.go +++ b/internal/arduino/libraries/librarieslist.go @@ -21,6 +21,7 @@ import ( "io" "sort" + "github.com/arduino/go-paths-helper" semver "go.bug.st/relaxed-semver" ) @@ -44,7 +45,7 @@ func (list *List) Add(libs ...*Library) { } } -func (list *List) UnmarshalBinary(in io.Reader) error { +func (list *List) UnmarshalBinary(in io.Reader, prefix *paths.Path) error { var n int32 if err := binary.Read(in, binary.NativeEndian, &n); err != nil { return err @@ -52,7 +53,7 @@ func (list *List) UnmarshalBinary(in io.Reader) error { res := make([]*Library, n) for i := range res { var lib Library - if err := lib.UnmarshalBinary(in); err != nil { + if err := lib.UnmarshalBinary(in, prefix); err != nil { return err } res[i] = &lib @@ -61,12 +62,12 @@ func (list *List) UnmarshalBinary(in io.Reader) error { return nil } -func (list *List) MarshalBinary(out io.Writer) error { +func (list *List) MarshalBinary(out io.Writer, prefix *paths.Path) error { if err := binary.Write(out, binary.NativeEndian, int32(len(*list))); err != nil { return err } for _, lib := range *list { - if err := lib.MarshalBinary(out); err != nil { + if err := lib.MarshalBinary(out, prefix); err != nil { return fmt.Errorf("could not encode lib data of %s: %w", lib.InstallDir.String(), err) } } diff --git a/internal/arduino/libraries/librariesmanager/librariesmanager.go b/internal/arduino/libraries/librariesmanager/librariesmanager.go index 6eba79c4e8d..8406e179011 100644 --- a/internal/arduino/libraries/librariesmanager/librariesmanager.go +++ b/internal/arduino/libraries/librariesmanager/librariesmanager.go @@ -213,7 +213,7 @@ func (lm *LibrariesManager) loadLibrariesFromDir(librariesDir *LibrariesDir) []* } defer cache.Close() - if err := loadedLibs.UnmarshalBinary(cache); err != nil { + if err := loadedLibs.UnmarshalBinary(cache, librariesDir.Path); err != nil { s := status.Newf(codes.FailedPrecondition, "reading lib cache %[1]s: %[2]s", cacheFilePath, err) return append(statuses, s) } @@ -251,7 +251,7 @@ func (lm *LibrariesManager) loadLibrariesFromDir(librariesDir *LibrariesDir) []* s := status.Newf(codes.FailedPrecondition, "creating lib cache %[1]s: %[2]s", cacheFilePath, err) return append(statuses, s) } - err = loadedLibs.MarshalBinary(cache) + err = loadedLibs.MarshalBinary(cache, librariesDir.Path) cache.Close() if err != nil { cacheFilePath.Remove() From a1360c48f191e4f490c825054419226c35772e32 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Thu, 3 Oct 2024 16:26:37 +0200 Subject: [PATCH 05/12] Do not allocate arrays if the saved array is empty --- internal/arduino/libraries/libraries.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/arduino/libraries/libraries.go b/internal/arduino/libraries/libraries.go index 3cee3afd046..06ebbaba0c8 100644 --- a/internal/arduino/libraries/libraries.go +++ b/internal/arduino/libraries/libraries.go @@ -249,6 +249,9 @@ func (library *Library) UnmarshalBinary(in io.Reader, prefix *paths.Path) error if err := binary.Read(in, binary.NativeEndian, &len); err != nil { return nil, err } + if len == 0 { + return nil, nil + } res := make([]string, len) for i := range res { var err error From 19f9fa47660b2642a7d1498323ed82976b7fccf2 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Thu, 3 Oct 2024 16:32:19 +0200 Subject: [PATCH 06/12] Cache library properties --- internal/arduino/libraries/libraries.go | 50 ++++++++++++++++++++----- 1 file changed, 41 insertions(+), 9 deletions(-) diff --git a/internal/arduino/libraries/libraries.go b/internal/arduino/libraries/libraries.go index 06ebbaba0c8..4a2ec00f1b7 100644 --- a/internal/arduino/libraries/libraries.go +++ b/internal/arduino/libraries/libraries.go @@ -126,6 +126,22 @@ func (library *Library) MarshalBinary(out io.Writer, prefix *paths.Path) error { } return nil } + writeProperties := func(in *properties.Map) error { + keys := in.Keys() + if err := binary.Write(out, binary.NativeEndian, uint16(len(keys))); err != nil { + return err + } + for _, k := range keys { + v := in.Get(k) + if err := writeString(k); err != nil { + return err + } + if err := writeString(v); err != nil { + return err + } + } + return nil + } writePath := func(in *paths.Path) error { if in == nil { return writeString("") @@ -216,7 +232,6 @@ func (library *Library) MarshalBinary(out io.Writer, prefix *paths.Path) error { if err := writeString(library.License); err != nil { return err } - //writeStringArray(library.Properties.AsSlice()) if err := writePathList(library.Examples); err != nil { return err } @@ -229,6 +244,10 @@ func (library *Library) MarshalBinary(out io.Writer, prefix *paths.Path) error { if err := writeMap(library.CompatibleWith); err != nil { return err } + if err := writeProperties(library.Properties); err != nil { + return err + } + return nil } @@ -305,6 +324,23 @@ func (library *Library) UnmarshalBinary(in io.Reader, prefix *paths.Path) error } return list, nil } + readProperties := func() (*properties.Map, error) { + var len uint16 + if err := binary.Read(in, binary.NativeEndian, &len); err != nil { + return nil, err + } + props := properties.NewMap() + for range len { + if k, err := readString(); err != nil { + return nil, err + } else if v, err := readString(); err != nil { + return nil, err + } else { + props.Set(k, v) + } + } + return props, nil + } var err error library.Name, err = readString() if err != nil { @@ -397,14 +433,6 @@ func (library *Library) UnmarshalBinary(in io.Reader, prefix *paths.Path) error if err != nil { return err } - // props, err := readStringArray() - // if err != nil { - // return err - // } - // library.Properties, err = properties.LoadFromSlice(props) - // if err != nil { - // return err - // } library.Examples, err = readPathList() if err != nil { return err @@ -421,6 +449,10 @@ func (library *Library) UnmarshalBinary(in io.Reader, prefix *paths.Path) error if err != nil { return err } + library.Properties, err = readProperties() + if err != nil { + return err + } return nil } From 9a1bec5e65fca689f873d0e585c5405542b4e805 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Thu, 3 Oct 2024 16:32:27 +0200 Subject: [PATCH 07/12] Nicer range --- internal/arduino/libraries/libraries.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/arduino/libraries/libraries.go b/internal/arduino/libraries/libraries.go index 4a2ec00f1b7..eb47a2ab84c 100644 --- a/internal/arduino/libraries/libraries.go +++ b/internal/arduino/libraries/libraries.go @@ -287,7 +287,7 @@ func (library *Library) UnmarshalBinary(in io.Reader, prefix *paths.Path) error return nil, err } res := map[string]bool{} - for i := uint16(0); i < len; i++ { + for range len { k, err := readString() if err != nil { return nil, err From 52e296126c0f104b9757a29abda9086ad0d8dc01 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Thu, 3 Oct 2024 16:54:21 +0200 Subject: [PATCH 08/12] Added magic number to binary encoder --- internal/arduino/libraries/librarieslist.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/internal/arduino/libraries/librarieslist.go b/internal/arduino/libraries/librarieslist.go index ff21a76bdb8..3043ed32f72 100644 --- a/internal/arduino/libraries/librarieslist.go +++ b/internal/arduino/libraries/librarieslist.go @@ -17,6 +17,7 @@ package libraries import ( "encoding/binary" + "errors" "fmt" "io" "sort" @@ -45,7 +46,18 @@ func (list *List) Add(libs ...*Library) { } } +func (list *List) binaryMagicNumber() uint32 { + return 0xAD000001 +} + func (list *List) UnmarshalBinary(in io.Reader, prefix *paths.Path) error { + var magic uint32 + if err := binary.Read(in, binary.NativeEndian, &magic); err != nil { + return err + } + if magic != list.binaryMagicNumber() { + return errors.New("invalid cache version") + } var n int32 if err := binary.Read(in, binary.NativeEndian, &n); err != nil { return err @@ -63,6 +75,9 @@ func (list *List) UnmarshalBinary(in io.Reader, prefix *paths.Path) error { } func (list *List) MarshalBinary(out io.Writer, prefix *paths.Path) error { + if err := binary.Write(out, binary.NativeEndian, list.binaryMagicNumber()); err != nil { + return err + } if err := binary.Write(out, binary.NativeEndian, int32(len(*list))); err != nil { return err } From 99694ef8aee9f187df92eefa6c3507fc1927972d Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Thu, 3 Oct 2024 16:54:37 +0200 Subject: [PATCH 09/12] Preload all source headers before writing the lib cache --- .../arduino/libraries/librariesmanager/librariesmanager.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/internal/arduino/libraries/librariesmanager/librariesmanager.go b/internal/arduino/libraries/librariesmanager/librariesmanager.go index 8406e179011..b4b7c43f54e 100644 --- a/internal/arduino/libraries/librariesmanager/librariesmanager.go +++ b/internal/arduino/libraries/librariesmanager/librariesmanager.go @@ -251,6 +251,11 @@ func (lm *LibrariesManager) loadLibrariesFromDir(librariesDir *LibrariesDir) []* s := status.Newf(codes.FailedPrecondition, "creating lib cache %[1]s: %[2]s", cacheFilePath, err) return append(statuses, s) } + // Preload source files and header + for _, lib := range loadedLibs { + lib.SourceHeaders() + } + // Write the cache err = loadedLibs.MarshalBinary(cache, librariesDir.Path) cache.Close() if err != nil { From bd37197679dbd425caeaf9e91aff5a34dde56854 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Wed, 9 Oct 2024 12:19:44 +0200 Subject: [PATCH 10/12] Handle nil properites --- internal/arduino/libraries/libraries.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/internal/arduino/libraries/libraries.go b/internal/arduino/libraries/libraries.go index eb47a2ab84c..025443488c3 100644 --- a/internal/arduino/libraries/libraries.go +++ b/internal/arduino/libraries/libraries.go @@ -127,7 +127,10 @@ func (library *Library) MarshalBinary(out io.Writer, prefix *paths.Path) error { return nil } writeProperties := func(in *properties.Map) error { - keys := in.Keys() + var keys []string + if in != nil { + keys = in.Keys() + } if err := binary.Write(out, binary.NativeEndian, uint16(len(keys))); err != nil { return err } From 0056058f85359e1c3925dd1426052f1312f5f06d Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Wed, 9 Oct 2024 12:19:58 +0200 Subject: [PATCH 11/12] Write relative paths only if path is inside prefix --- internal/arduino/libraries/libraries.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/internal/arduino/libraries/libraries.go b/internal/arduino/libraries/libraries.go index 025443488c3..7c2cb39fdca 100644 --- a/internal/arduino/libraries/libraries.go +++ b/internal/arduino/libraries/libraries.go @@ -20,6 +20,7 @@ import ( "errors" "fmt" "io" + "path/filepath" "github.com/arduino/arduino-cli/internal/arduino/cores" "github.com/arduino/arduino-cli/internal/arduino/globals" @@ -148,6 +149,10 @@ func (library *Library) MarshalBinary(out io.Writer, prefix *paths.Path) error { writePath := func(in *paths.Path) error { if in == nil { return writeString("") + } else if inside, err := in.IsInsideDir(prefix); err != nil { + return err + } else if !inside { + return writeString(in.String()) } else if p, err := in.RelFrom(prefix); err != nil { return err } else { @@ -308,6 +313,8 @@ func (library *Library) UnmarshalBinary(in io.Reader, prefix *paths.Path) error return nil, err } else if p == "" { return nil, nil + } else if filepath.IsAbs(p) { + return paths.New(p), nil } else { return prefix.Join(p), nil } From bb73b9cbccada29808a879f2967dea38f26a6bdc Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Wed, 30 Oct 2024 09:16:18 +0100 Subject: [PATCH 12/12] Do not cache lib-scan for 'unmanaged' paths Those are the paths given by --library / --libraries flags. --- .../librariesmanager/librariesmanager.go | 27 ++++++++++--------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/internal/arduino/libraries/librariesmanager/librariesmanager.go b/internal/arduino/libraries/librariesmanager/librariesmanager.go index b4b7c43f54e..53875e48cc3 100644 --- a/internal/arduino/libraries/librariesmanager/librariesmanager.go +++ b/internal/arduino/libraries/librariesmanager/librariesmanager.go @@ -245,23 +245,24 @@ func (lm *LibrariesManager) loadLibrariesFromDir(librariesDir *LibrariesDir) []* loadedLibs = append(loadedLibs, library) } - // Write lib cache - cache, err := cacheFilePath.Create() - if err != nil { - s := status.Newf(codes.FailedPrecondition, "creating lib cache %[1]s: %[2]s", cacheFilePath, err) - return append(statuses, s) - } // Preload source files and header for _, lib := range loadedLibs { lib.SourceHeaders() } - // Write the cache - err = loadedLibs.MarshalBinary(cache, librariesDir.Path) - cache.Close() - if err != nil { - cacheFilePath.Remove() - s := status.Newf(codes.FailedPrecondition, "writing lib cache %[1]s: %[2]s", cacheFilePath, err) - return append(statuses, s) + if librariesDir.Location != libraries.Unmanaged { + // Write lib cache + cache, err := cacheFilePath.Create() + if err != nil { + s := status.Newf(codes.FailedPrecondition, "creating lib cache %[1]s: %[2]s", cacheFilePath, err) + return append(statuses, s) + } + err = loadedLibs.MarshalBinary(cache, librariesDir.Path) + cache.Close() + if err != nil { + cacheFilePath.Remove() + s := status.Newf(codes.FailedPrecondition, "writing lib cache %[1]s: %[2]s", cacheFilePath, err) + return append(statuses, s) + } } } pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy