From cae59be4bbe10b0fd53ccbeb613641f9d61c6da0 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Tue, 15 Oct 2024 00:21:59 +0200 Subject: [PATCH 1/4] Increased logging during compile --- internal/arduino/builder/internal/utils/utils.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/internal/arduino/builder/internal/utils/utils.go b/internal/arduino/builder/internal/utils/utils.go index 630172e2410..bf5b92711e0 100644 --- a/internal/arduino/builder/internal/utils/utils.go +++ b/internal/arduino/builder/internal/utils/utils.go @@ -32,13 +32,14 @@ import ( func ObjFileIsUpToDate(sourceFile, objectFile, dependencyFile *paths.Path) (bool, error) { logrus.Debugf("Checking previous results for %v (result = %v, dep = %v)", sourceFile, objectFile, dependencyFile) if objectFile == nil || dependencyFile == nil { - logrus.Debugf("Not found: nil") + logrus.Debugf("Object file or dependency file not provided") return false, nil } sourceFile = sourceFile.Clean() sourceFileStat, err := sourceFile.Stat() if err != nil { + logrus.Debugf("Could not stat source file: %s", err) return false, err } @@ -46,9 +47,10 @@ func ObjFileIsUpToDate(sourceFile, objectFile, dependencyFile *paths.Path) (bool objectFileStat, err := objectFile.Stat() if err != nil { if os.IsNotExist(err) { - logrus.Debugf("Not found: %v", objectFile) + logrus.Debugf("Object file not found: %v", objectFile) return false, nil } + logrus.Debugf("Could not stat object file: %s", err) return false, err } @@ -56,9 +58,10 @@ func ObjFileIsUpToDate(sourceFile, objectFile, dependencyFile *paths.Path) (bool dependencyFileStat, err := dependencyFile.Stat() if err != nil { if os.IsNotExist(err) { - logrus.Debugf("Not found: %v", dependencyFile) + logrus.Debugf("Dependency file not found: %v", dependencyFile) return false, nil } + logrus.Debugf("Could not stat dependency file: %s", err) return false, err } @@ -73,6 +76,7 @@ func ObjFileIsUpToDate(sourceFile, objectFile, dependencyFile *paths.Path) (bool rows, err := dependencyFile.ReadFileAsLines() if err != nil { + logrus.Debugf("Could not read dependency file: %s", dependencyFile) return false, err } @@ -92,7 +96,7 @@ func ObjFileIsUpToDate(sourceFile, objectFile, dependencyFile *paths.Path) (bool } objFileInDepFile := firstRow[:len(firstRow)-1] if objFileInDepFile != objectFile.String() { - logrus.Debugf("Depfile is about different file: %v", objFileInDepFile) + logrus.Debugf("Depfile is about different object file: %v", objFileInDepFile) return false, nil } @@ -103,6 +107,7 @@ func ObjFileIsUpToDate(sourceFile, objectFile, dependencyFile *paths.Path) (bool // If we don't do this check it might happen that trying to compile a source file // that has the same name but a different path wouldn't recreate the object file. if sourceFile.String() != strings.Trim(rows[1], " ") { + logrus.Debugf("Depfile is about different source file: %v", strings.Trim(rows[1], " ")) return false, nil } From c1b290ac1011ea35f6134722a9d77c658f95caa9 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Tue, 15 Oct 2024 11:39:39 +0200 Subject: [PATCH 2/4] Added integration test --- .../compile_4/lib_caching_test.go | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 internal/integrationtest/compile_4/lib_caching_test.go diff --git a/internal/integrationtest/compile_4/lib_caching_test.go b/internal/integrationtest/compile_4/lib_caching_test.go new file mode 100644 index 00000000000..1a497da7747 --- /dev/null +++ b/internal/integrationtest/compile_4/lib_caching_test.go @@ -0,0 +1,60 @@ +// This file is part of arduino-cli. +// +// Copyright 2024 ARDUINO SA (http://www.arduino.cc/) +// +// This software is released under the GNU General Public License version 3, +// which covers the main part of arduino-cli. +// The terms of this license can be found at: +// https://www.gnu.org/licenses/gpl-3.0.en.html +// +// You can be released from the requirements of the above licenses by purchasing +// a commercial license. Buying such a license is mandatory if you want to +// modify or otherwise use the software for commercial activities involving the +// Arduino software without disclosing the source code of your own applications. +// To purchase a commercial license, send an email to license@arduino.cc. + +package compile + +import ( + "testing" + + "github.com/arduino/arduino-cli/internal/integrationtest" + "github.com/arduino/go-paths-helper" + "github.com/stretchr/testify/require" +) + +func TestBuildCacheLibWithNonASCIIChars(t *testing.T) { + // See: https://github.com/arduino/arduino-cli/issues/2671 + + env, cli := integrationtest.CreateArduinoCLIWithEnvironment(t) + t.Cleanup(env.CleanUp) + + tmpUserDir, err := paths.MkTempDir("", "Håkan") + require.NoError(t, err) + t.Cleanup(func() { tmpUserDir.RemoveAll() }) + customEnv := cli.GetDefaultEnv() + customEnv["ARDUINO_DIRECTORIES_USER"] = tmpUserDir.String() + + // Install Arduino AVR Boards and Servo lib + _, _, err = cli.Run("core", "install", "arduino:avr@1.8.6") + require.NoError(t, err) + _, _, err = cli.RunWithCustomEnv(customEnv, "lib", "install", "Servo") + require.NoError(t, err) + + // Make a temp sketch + sketchDir := tmpUserDir.Join("ServoSketch") + sketchFile := sketchDir.Join("ServoSketch.ino") + require.NoError(t, sketchDir.Mkdir()) + require.NoError(t, sketchFile.WriteFile( + []byte("#include \nvoid setup() {}\nvoid loop() {}\n"), + )) + + // Compile sketch + _, _, err = cli.RunWithCustomEnv(customEnv, "compile", "-b", "arduino:avr:uno", sketchFile.String()) + require.NoError(t, err) + + // Compile sketch again + out, _, err := cli.RunWithCustomEnv(customEnv, "compile", "-b", "arduino:avr:uno", "-v", sketchFile.String()) + require.NoError(t, err) + require.Contains(t, string(out), "Compiling library \"Servo\"\nUsing previously compiled file") +} From cff613fcfec47c628acf3cbd6214546e0ff3a8a5 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Wed, 16 Oct 2024 10:56:27 +0200 Subject: [PATCH 3/4] Fixed dependency file parsing on Windows --- go.mod | 2 +- .../builder/internal/utils/ansi_others.go | 27 +++++ .../builder/internal/utils/ansi_windows.go | 33 ++++++ .../arduino/builder/internal/utils/utils.go | 101 ++++++++++-------- 4 files changed, 120 insertions(+), 43 deletions(-) create mode 100644 internal/arduino/builder/internal/utils/ansi_others.go create mode 100644 internal/arduino/builder/internal/utils/ansi_windows.go diff --git a/go.mod b/go.mod index 93fb125a868..59a9a96337e 100644 --- a/go.mod +++ b/go.mod @@ -39,6 +39,7 @@ require ( go.bug.st/f v0.4.0 go.bug.st/relaxed-semver v0.12.0 go.bug.st/testifyjson v1.2.0 + golang.org/x/sys v0.26.0 golang.org/x/term v0.25.0 golang.org/x/text v0.19.0 google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142 @@ -100,7 +101,6 @@ require ( golang.org/x/mod v0.18.0 // indirect golang.org/x/net v0.28.0 // indirect golang.org/x/sync v0.8.0 // indirect - golang.org/x/sys v0.26.0 // indirect golang.org/x/tools v0.22.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect diff --git a/internal/arduino/builder/internal/utils/ansi_others.go b/internal/arduino/builder/internal/utils/ansi_others.go new file mode 100644 index 00000000000..a49a7bb1b25 --- /dev/null +++ b/internal/arduino/builder/internal/utils/ansi_others.go @@ -0,0 +1,27 @@ +// This file is part of arduino-cli. +// +// Copyright 2024 ARDUINO SA (http://www.arduino.cc/) +// +// This software is released under the GNU General Public License version 3, +// which covers the main part of arduino-cli. +// The terms of this license can be found at: +// https://www.gnu.org/licenses/gpl-3.0.en.html +// +// You can be released from the requirements of the above licenses by purchasing +// a commercial license. Buying such a license is mandatory if you want to +// modify or otherwise use the software for commercial activities involving the +// Arduino software without disclosing the source code of your own applications. +// To purchase a commercial license, send an email to license@arduino.cc. + +//go:build !windows + +package utils + +import ( + "errors" +) + +// placeholder for non-Windows machines +func convertAnsiBytesToString([]byte) (string, error) { + return "", errors.New("unimplemented") +} diff --git a/internal/arduino/builder/internal/utils/ansi_windows.go b/internal/arduino/builder/internal/utils/ansi_windows.go new file mode 100644 index 00000000000..1c0999f1687 --- /dev/null +++ b/internal/arduino/builder/internal/utils/ansi_windows.go @@ -0,0 +1,33 @@ +// This file is part of arduino-cli. +// +// Copyright 2024 ARDUINO SA (http://www.arduino.cc/) +// +// This software is released under the GNU General Public License version 3, +// which covers the main part of arduino-cli. +// The terms of this license can be found at: +// https://www.gnu.org/licenses/gpl-3.0.en.html +// +// You can be released from the requirements of the above licenses by purchasing +// a commercial license. Buying such a license is mandatory if you want to +// modify or otherwise use the software for commercial activities involving the +// Arduino software without disclosing the source code of your own applications. +// To purchase a commercial license, send an email to license@arduino.cc. + +package utils + +import ( + "golang.org/x/sys/windows" +) + +func convertAnsiBytesToString(data []byte) (string, error) { + dataSize := int32(len(data)) + size, err := windows.MultiByteToWideChar(windows.GetACP(), 0, &data[0], dataSize, nil, 0) + if err != nil { + return "", err + } + utf16 := make([]uint16, size) + if _, err := windows.MultiByteToWideChar(windows.GetACP(), 0, &data[0], dataSize, &utf16[0], size); err != nil { + return "", err + } + return windows.UTF16ToString(utf16), nil +} diff --git a/internal/arduino/builder/internal/utils/utils.go b/internal/arduino/builder/internal/utils/utils.go index bf5b92711e0..d0cb2cec48c 100644 --- a/internal/arduino/builder/internal/utils/utils.go +++ b/internal/arduino/builder/internal/utils/utils.go @@ -17,6 +17,7 @@ package utils import ( "os" + "runtime" "strings" "unicode" @@ -74,63 +75,79 @@ func ObjFileIsUpToDate(sourceFile, objectFile, dependencyFile *paths.Path) (bool return false, nil } - rows, err := dependencyFile.ReadFileAsLines() + depFileData, err := dependencyFile.ReadFile() if err != nil { logrus.Debugf("Could not read dependency file: %s", dependencyFile) return false, err } - rows = f.Map(rows, removeEndingBackSlash) - rows = f.Map(rows, strings.TrimSpace) - rows = f.Map(rows, unescapeDep) - rows = f.Filter(rows, f.NotEquals("")) + checkDepFile := func(depFile string) (bool, error) { + rows := strings.Split(strings.Replace(depFile, "\r\n", "\n", -1), "\n") + rows = f.Map(rows, removeEndingBackSlash) + rows = f.Map(rows, strings.TrimSpace) + rows = f.Map(rows, unescapeDep) + rows = f.Filter(rows, f.NotEquals("")) - if len(rows) == 0 { - return true, nil - } - - firstRow := rows[0] - if !strings.HasSuffix(firstRow, ":") { - logrus.Debugf("No colon in first line of depfile") - return false, nil - } - objFileInDepFile := firstRow[:len(firstRow)-1] - if objFileInDepFile != objectFile.String() { - logrus.Debugf("Depfile is about different object file: %v", objFileInDepFile) - return false, nil - } - - // The first line of the depfile contains the path to the object file to generate. - // The second line of the depfile contains the path to the source file. - // All subsequent lines contain the header files necessary to compile the object file. - - // If we don't do this check it might happen that trying to compile a source file - // that has the same name but a different path wouldn't recreate the object file. - if sourceFile.String() != strings.Trim(rows[1], " ") { - logrus.Debugf("Depfile is about different source file: %v", strings.Trim(rows[1], " ")) - return false, nil - } + if len(rows) == 0 { + return true, nil + } - rows = rows[1:] - for _, row := range rows { - depStat, err := os.Stat(row) - if err != nil && !os.IsNotExist(err) { - // There is probably a parsing error of the dep file - // Ignore the error and trigger a full rebuild anyway - logrus.WithError(err).Debugf("Failed to read: %v", row) + firstRow := rows[0] + if !strings.HasSuffix(firstRow, ":") { + logrus.Debugf("No colon in first line of depfile") return false, nil } - if os.IsNotExist(err) { - logrus.Debugf("Not found: %v", row) + objFileInDepFile := firstRow[:len(firstRow)-1] + if objFileInDepFile != objectFile.String() { + logrus.Debugf("Depfile is about different object file: %v", objFileInDepFile) return false, nil } - if depStat.ModTime().After(objectFileStat.ModTime()) { - logrus.Debugf("%v newer than %v", row, objectFile) + + // The first line of the depfile contains the path to the object file to generate. + // The second line of the depfile contains the path to the source file. + // All subsequent lines contain the header files necessary to compile the object file. + + // If we don't do this check it might happen that trying to compile a source file + // that has the same name but a different path wouldn't recreate the object file. + if sourceFile.String() != strings.Trim(rows[1], " ") { + logrus.Debugf("Depfile is about different source file: %v", strings.Trim(rows[1], " ")) return false, nil } + + rows = rows[1:] + for _, row := range rows { + depStat, err := os.Stat(row) + if err != nil && !os.IsNotExist(err) { + // There is probably a parsing error of the dep file + // Ignore the error and trigger a full rebuild anyway + logrus.WithError(err).Debugf("Failed to read: %v", row) + return false, nil + } + if os.IsNotExist(err) { + logrus.Debugf("Not found: %v", row) + return false, nil + } + if depStat.ModTime().After(objectFileStat.ModTime()) { + logrus.Debugf("%v newer than %v", row, objectFile) + return false, nil + } + } + + return true, nil } - return true, nil + if runtime.GOOS == "windows" { + // This is required because on Windows we don't know which encoding is used + // by gcc to write the dep file (it could be UTF-8 or any of the Windows + // ANSI mappings). + if decoded, err := convertAnsiBytesToString(depFileData); err == nil { + if upToDate, err := checkDepFile(decoded); err == nil && upToDate { + return upToDate, nil + } + } + // Fallback to UTF-8... + } + return checkDepFile(string(depFileData)) } func removeEndingBackSlash(s string) string { From 91c7fb39d19a41ea3a8c0ecdae39a1d15c7ea69b Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Wed, 16 Oct 2024 11:09:37 +0200 Subject: [PATCH 4/4] Fixed panic in convertAnsiBytesToString implementation --- internal/arduino/builder/internal/utils/ansi_windows.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/arduino/builder/internal/utils/ansi_windows.go b/internal/arduino/builder/internal/utils/ansi_windows.go index 1c0999f1687..248e7657b6f 100644 --- a/internal/arduino/builder/internal/utils/ansi_windows.go +++ b/internal/arduino/builder/internal/utils/ansi_windows.go @@ -20,6 +20,9 @@ import ( ) func convertAnsiBytesToString(data []byte) (string, error) { + if len(data) == 0 { + return "", nil + } dataSize := int32(len(data)) size, err := windows.MultiByteToWideChar(windows.GetACP(), 0, &data[0], dataSize, nil, 0) if err != nil { pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy